From ccc478fbaa986b46af31d130ffa280a2af9e0113 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Fri, 13 Mar 2026 14:29:38 -0300 Subject: [PATCH 1/3] chunker and ui --- .dockerignore | 30 + admin/mpr/celery.py | 2 +- admin/mpr/media_assets/models.py | 40 + core/api/chunker_sse.py | 78 + core/api/graphql.py | 89 +- core/api/main.py | 4 + core/api/schema/graphql.py | 49 + core/chunker/__init__.py | 64 + core/chunker/chunker.py | 86 + core/chunker/collector.py | 98 + core/chunker/exceptions.py | 64 + core/chunker/models.py | 54 + core/chunker/pipeline.py | 244 +++ core/chunker/pool.py | 125 ++ core/chunker/processor.py | 173 ++ core/chunker/queue.py | 76 + core/chunker/worker.py | 141 ++ core/jobs/__init__.py | 15 + core/{task => jobs}/executor.py | 128 +- core/{task => jobs}/gcp_handler.py | 0 core/jobs/handlers/__init__.py | 5 + core/jobs/handlers/base.py | 33 + core/jobs/handlers/chunk.py | 119 ++ core/jobs/handlers/transcode.py | 104 + core/{task => jobs}/lambda_handler.py | 0 core/jobs/registry.py | 33 + core/jobs/task.py | 64 + core/rpc/server.py | 27 +- core/schema/models/__init__.py | 8 +- core/schema/models/jobs.py | 60 +- core/task/__init__.py | 15 - core/task/tasks.py | 105 - ctrl/Dockerfile | 3 +- ctrl/Dockerfile.worker | 3 +- ctrl/docker-compose.yml | 49 +- ctrl/lambda/Dockerfile | 6 +- tests/__init__.py | 0 tests/chunker/__init__.py | 0 tests/chunker/conftest.py | 76 + tests/chunker/test_chunker.py | 149 ++ tests/chunker/test_collector.py | 103 + tests/chunker/test_exceptions.py | 69 + tests/chunker/test_pipeline.py | 144 ++ tests/chunker/test_processor.py | 98 + tests/chunker/test_queue.py | 115 ++ tests/chunker/test_worker.py | 127 ++ ui/chunker/index.html | 12 + ui/chunker/package-lock.json | 1729 +++++++++++++++++ ui/chunker/package.json | 22 + ui/chunker/src/App.css | 735 +++++++ ui/chunker/src/App.tsx | 245 +++ ui/chunker/src/api.ts | 72 + ui/chunker/src/components/ChunkGrid.tsx | 59 + ui/chunker/src/components/ConfigPanel.tsx | 172 ++ ui/chunker/src/components/ErrorLog.tsx | 63 + ui/chunker/src/components/PipelineDiagram.tsx | 50 + ui/chunker/src/components/QueueGauge.tsx | 46 + ui/chunker/src/components/StatsPanel.tsx | 59 + ui/chunker/src/components/TopicBadge.tsx | 86 + ui/chunker/src/components/WorkerPanel.tsx | 55 + ui/chunker/src/hooks/useEventStream.ts | 81 + ui/chunker/src/main.tsx | 9 + ui/chunker/src/types.ts | 114 ++ ui/chunker/src/vite-env.d.ts | 1 + ui/chunker/tsconfig.json | 21 + ui/chunker/tsconfig.node.json | 10 + ui/chunker/vite.config.ts | 21 + ui/timeline/.dockerignore | 2 + ui/timeline/src/types.ts | 24 + 69 files changed, 6481 insertions(+), 282 deletions(-) create mode 100644 .dockerignore create mode 100644 core/api/chunker_sse.py create mode 100644 core/chunker/__init__.py create mode 100644 core/chunker/chunker.py create mode 100644 core/chunker/collector.py create mode 100644 core/chunker/exceptions.py create mode 100644 core/chunker/models.py create mode 100644 core/chunker/pipeline.py create mode 100644 core/chunker/pool.py create mode 100644 core/chunker/processor.py create mode 100644 core/chunker/queue.py create mode 100644 core/chunker/worker.py create mode 100644 core/jobs/__init__.py rename core/{task => jobs}/executor.py (53%) rename core/{task => jobs}/gcp_handler.py (100%) create mode 100644 core/jobs/handlers/__init__.py create mode 100644 core/jobs/handlers/base.py create mode 100644 core/jobs/handlers/chunk.py create mode 100644 core/jobs/handlers/transcode.py rename core/{task => jobs}/lambda_handler.py (100%) create mode 100644 core/jobs/registry.py create mode 100644 core/jobs/task.py delete mode 100644 core/task/__init__.py delete mode 100644 core/task/tasks.py create mode 100644 tests/__init__.py create mode 100644 tests/chunker/__init__.py create mode 100644 tests/chunker/conftest.py create mode 100644 tests/chunker/test_chunker.py create mode 100644 tests/chunker/test_collector.py create mode 100644 tests/chunker/test_exceptions.py create mode 100644 tests/chunker/test_pipeline.py create mode 100644 tests/chunker/test_processor.py create mode 100644 tests/chunker/test_queue.py create mode 100644 tests/chunker/test_worker.py create mode 100644 ui/chunker/index.html create mode 100644 ui/chunker/package-lock.json create mode 100644 ui/chunker/package.json create mode 100644 ui/chunker/src/App.css create mode 100644 ui/chunker/src/App.tsx create mode 100644 ui/chunker/src/api.ts create mode 100644 ui/chunker/src/components/ChunkGrid.tsx create mode 100644 ui/chunker/src/components/ConfigPanel.tsx create mode 100644 ui/chunker/src/components/ErrorLog.tsx create mode 100644 ui/chunker/src/components/PipelineDiagram.tsx create mode 100644 ui/chunker/src/components/QueueGauge.tsx create mode 100644 ui/chunker/src/components/StatsPanel.tsx create mode 100644 ui/chunker/src/components/TopicBadge.tsx create mode 100644 ui/chunker/src/components/WorkerPanel.tsx create mode 100644 ui/chunker/src/hooks/useEventStream.ts create mode 100644 ui/chunker/src/main.tsx create mode 100644 ui/chunker/src/types.ts create mode 100644 ui/chunker/src/vite-env.d.ts create mode 100644 ui/chunker/tsconfig.json create mode 100644 ui/chunker/tsconfig.node.json create mode 100644 ui/chunker/vite.config.ts create mode 100644 ui/timeline/.dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..78ee4c1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +# Python +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +.pytest_cache/ + +# Node +node_modules/ +ui/*/node_modules/ +ui/*/dist/ + +# Media (9.8GB — mounted via volume, never needed in image) +media/ + +# Git +.git/ + +# IDE / OS +.idea/ +.vscode/ +*.swp +.DS_Store + +# Docker +ctrl/docker-compose.yml + +# Docs +docs/ +*.md diff --git a/admin/mpr/celery.py b/admin/mpr/celery.py index 93c2776..358c62e 100644 --- a/admin/mpr/celery.py +++ b/admin/mpr/celery.py @@ -7,4 +7,4 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "admin.mpr.settings") app = Celery("mpr") app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() -app.autodiscover_tasks(["core.task"]) +app.autodiscover_tasks(["core.jobs"]) diff --git a/admin/mpr/media_assets/models.py b/admin/mpr/media_assets/models.py index 2ee9e29..6576093 100644 --- a/admin/mpr/media_assets/models.py +++ b/admin/mpr/media_assets/models.py @@ -19,6 +19,15 @@ class JobStatus(models.TextChoices): FAILED = "failed", "Failed" CANCELLED = "cancelled", "Cancelled" +class ChunkJobStatus(models.TextChoices): + PENDING = "pending", "Pending" + CHUNKING = "chunking", "Chunking" + PROCESSING = "processing", "Processing" + COLLECTING = "collecting", "Collecting" + COMPLETED = "completed", "Completed" + FAILED = "failed", "Failed" + CANCELLED = "cancelled", "Cancelled" + class MediaAsset(models.Model): """A video/audio file registered in the system.""" @@ -108,3 +117,34 @@ class TranscodeJob(models.Model): def __str__(self): return str(self.id) + +class ChunkJob(models.Model): + """A chunk pipeline job — splits a media file into chunks and processes them""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + source_asset_id = models.UUIDField() + chunk_duration = models.FloatField(default=10.0) + num_workers = models.IntegerField(default=4) + max_retries = models.IntegerField(default=3) + processor_type = models.CharField(max_length=255) + status = models.CharField(max_length=20, choices=ChunkJobStatus.choices, default=ChunkJobStatus.PENDING) + progress = models.FloatField(default=0.0) + total_chunks = models.IntegerField(default=0) + processed_chunks = models.IntegerField(default=0) + failed_chunks = models.IntegerField(default=0) + retry_count = models.IntegerField(default=0) + error_message = models.TextField(blank=True, default='') + throughput_mbps = models.FloatField(null=True, blank=True, default=None) + elapsed_seconds = models.FloatField(null=True, blank=True, default=None) + celery_task_id = models.CharField(max_length=255, null=True, blank=True) + priority = models.IntegerField(default=0) + created_at = models.DateTimeField(auto_now_add=True) + started_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ["-created_at"] + + def __str__(self): + return str(self.id) + diff --git a/core/api/chunker_sse.py b/core/api/chunker_sse.py new file mode 100644 index 0000000..5090ab1 --- /dev/null +++ b/core/api/chunker_sse.py @@ -0,0 +1,78 @@ +""" +SSE endpoint for chunker pipeline events. + +Bridges gRPC StreamProgress to browser-native EventSource. +GET /api/chunker/stream/{job_id} → text/event-stream +""" + +import asyncio +import json +import logging +import time +from typing import AsyncGenerator + +from fastapi import APIRouter +from starlette.responses import StreamingResponse + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/chunker", tags=["chunker"]) + + +async def _event_generator(job_id: str) -> AsyncGenerator[str, None]: + """ + Generate SSE events by polling gRPC job state. + + Yields server-sent events in the format: + event: + data: + """ + from core.rpc.server import _active_jobs + + last_state = None + timeout = time.monotonic() + 600 # 10 min max + + while time.monotonic() < timeout: + job_state = _active_jobs.get(job_id) + + if job_state is None: + # Job not found yet — may not have started + yield f"event: waiting\ndata: {json.dumps({'job_id': job_id})}\n\n" + await asyncio.sleep(0.5) + continue + + # Only send if state changed + if job_state != last_state: + last_state = dict(job_state) + event_type = job_state.get("status", "update") + + yield f"event: {event_type}\ndata: {json.dumps({**job_state, 'job_id': job_id})}\n\n" + + # End stream when job is terminal + if event_type in ("completed", "failed", "cancelled"): + yield f"event: done\ndata: {json.dumps({'job_id': job_id})}\n\n" + break + + await asyncio.sleep(0.2) + + yield f"event: timeout\ndata: {json.dumps({'job_id': job_id})}\n\n" + + +@router.get("/stream/{job_id}") +async def stream_chunk_job(job_id: str): + """ + SSE stream for a chunk pipeline job. + + The UI connects via native EventSource: + const es = new EventSource('/api/chunker/stream/'); + es.addEventListener('processing', (e) => { ... }); + """ + return StreamingResponse( + _event_generator(job_id), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) diff --git a/core/api/graphql.py b/core/api/graphql.py index 7a88e8f..15ea60d 100644 --- a/core/api/graphql.py +++ b/core/api/graphql.py @@ -15,6 +15,8 @@ from strawberry.schema.config import StrawberryConfig from strawberry.types import Info from core.api.schema.graphql import ( + ChunkJobType, + CreateChunkJobInput, CreateJobInput, DeleteResultType, MediaAssetType, @@ -172,30 +174,31 @@ class Mutation: priority=input.priority or 0, ) + payload = { + "source_key": source.file_path, + "output_key": output_filename, + "preset": preset_snapshot or None, + "trim_start": input.trim_start, + "trim_end": input.trim_end, + "duration": source.duration, + } + executor_mode = os.environ.get("MPR_EXECUTOR", "local") if executor_mode in ("lambda", "gcp"): - from core.task.executor import get_executor + from core.jobs.executor import get_executor get_executor().run( + job_type="transcode", job_id=str(job.id), - source_path=source.file_path, - output_path=output_filename, - preset=preset_snapshot or None, - trim_start=input.trim_start, - trim_end=input.trim_end, - duration=source.duration, + payload=payload, ) else: - from core.task.tasks import run_transcode_job + from core.jobs.task import run_job - result = run_transcode_job.delay( + result = run_job.delay( + job_type="transcode", job_id=str(job.id), - source_key=source.file_path, - output_key=output_filename, - preset=preset_snapshot or None, - trim_start=input.trim_start, - trim_end=input.trim_end, - duration=source.duration, + payload=payload, ) job.celery_task_id = result.id job.save(update_fields=["celery_task_id"]) @@ -261,6 +264,62 @@ class Mutation: except Exception: raise Exception("Asset not found") + @strawberry.mutation + def create_chunk_job(self, info: Info, input: CreateChunkJobInput) -> ChunkJobType: + """Create and dispatch a chunk pipeline job.""" + import uuid + + from core.db import get_asset + + try: + source = get_asset(input.source_asset_id) + except Exception: + raise Exception("Source asset not found") + + job_id = str(uuid.uuid4()) + + payload = { + "source_key": source.file_path, + "chunk_duration": input.chunk_duration, + "num_workers": input.num_workers, + "max_retries": input.max_retries, + "processor_type": input.processor_type, + } + + executor_mode = os.environ.get("MPR_EXECUTOR", "local") + celery_task_id = None + + if executor_mode in ("lambda", "gcp"): + from core.jobs.executor import get_executor + + get_executor().run( + job_type="chunk", + job_id=job_id, + payload=payload, + ) + else: + from core.jobs.task import run_job + + result = run_job.delay( + job_type="chunk", + job_id=job_id, + payload=payload, + ) + celery_task_id = result.id + + return ChunkJobType( + id=uuid.UUID(job_id), + source_asset_id=input.source_asset_id, + chunk_duration=input.chunk_duration, + num_workers=input.num_workers, + max_retries=input.max_retries, + processor_type=input.processor_type, + status="pending", + progress=0.0, + priority=input.priority, + celery_task_id=celery_task_id, + ) + # --------------------------------------------------------------------------- # Schema diff --git a/core/api/main.py b/core/api/main.py index 6844a69..8d62cf8 100644 --- a/core/api/main.py +++ b/core/api/main.py @@ -23,6 +23,7 @@ from fastapi import FastAPI, Header, HTTPException from fastapi.middleware.cors import CORSMiddleware from strawberry.fastapi import GraphQLRouter +from core.api.chunker_sse import router as chunker_router from core.api.graphql import schema as graphql_schema CALLBACK_API_KEY = os.environ.get("CALLBACK_API_KEY", "") @@ -48,6 +49,9 @@ app.add_middleware( graphql_router = GraphQLRouter(schema=graphql_schema, graphql_ide="graphiql") app.include_router(graphql_router, prefix="/graphql") +# Chunker SSE +app.include_router(chunker_router) + @app.get("/") def root(): diff --git a/core/api/schema/graphql.py b/core/api/schema/graphql.py index 48e2eb4..51f1bda 100644 --- a/core/api/schema/graphql.py +++ b/core/api/schema/graphql.py @@ -156,3 +156,52 @@ class WorkerStatusType: active_jobs: Optional[int] = None supported_codecs: Optional[List[str]] = None gpu_available: Optional[bool] = None + + +@strawberry.enum +class ChunkJobStatus(Enum): + PENDING = "pending" + CHUNKING = "chunking" + PROCESSING = "processing" + COLLECTING = "collecting" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@strawberry.type +class ChunkJobType: + """A chunk pipeline job.""" + + id: Optional[UUID] = None + source_asset_id: Optional[UUID] = None + chunk_duration: Optional[float] = None + num_workers: Optional[int] = None + max_retries: Optional[int] = None + processor_type: Optional[str] = None + status: Optional[str] = None + progress: Optional[float] = None + total_chunks: Optional[int] = None + processed_chunks: Optional[int] = None + failed_chunks: Optional[int] = None + retry_count: Optional[int] = None + error_message: Optional[str] = None + throughput_mbps: Optional[float] = None + elapsed_seconds: Optional[float] = None + celery_task_id: Optional[str] = None + priority: Optional[int] = None + created_at: Optional[datetime] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + +@strawberry.input +class CreateChunkJobInput: + """Request body for creating a chunk pipeline job.""" + + source_asset_id: UUID + chunk_duration: float = 10.0 + num_workers: int = 4 + max_retries: int = 3 + processor_type: str = "ffmpeg" + priority: int = 0 diff --git a/core/chunker/__init__.py b/core/chunker/__init__.py new file mode 100644 index 0000000..81effee --- /dev/null +++ b/core/chunker/__init__.py @@ -0,0 +1,64 @@ +""" +Chunker pipeline — splits files into chunks, processes concurrently, reassembles in order. + +Public API: + Pipeline — orchestrates the full pipeline + PipelineResult — aggregate result dataclass + Chunker — file → Chunk generator + ChunkQueue — bounded thread-safe queue + WorkerPool — manages N worker threads + ResultCollector — heapq-based ordered reassembly +""" + +from .chunker import Chunker +from .collector import ResultCollector +from .exceptions import ( + ChunkChecksumError, + ChunkError, + ChunkReadError, + PipelineError, + ProcessingError, + ProcessorFailureError, + ProcessorTimeoutError, + ReassemblyError, +) +from .models import Chunk, ChunkResult, PipelineResult +from .pipeline import Pipeline +from .pool import WorkerPool +from .processor import ( + ChecksumProcessor, + CompositeProcessor, + FFmpegExtractProcessor, + Processor, + SimulatedDecodeProcessor, +) +from .queue import ChunkQueue + +__all__ = [ + # Core + "Pipeline", + "PipelineResult", + # Components + "Chunker", + "ChunkQueue", + "WorkerPool", + "ResultCollector", + # Models + "Chunk", + "ChunkResult", + # Processors + "Processor", + "ChecksumProcessor", + "SimulatedDecodeProcessor", + "CompositeProcessor", + "FFmpegExtractProcessor", + # Exceptions + "PipelineError", + "ChunkError", + "ChunkReadError", + "ChunkChecksumError", + "ProcessingError", + "ProcessorFailureError", + "ProcessorTimeoutError", + "ReassemblyError", +] diff --git a/core/chunker/chunker.py b/core/chunker/chunker.py new file mode 100644 index 0000000..8301c17 --- /dev/null +++ b/core/chunker/chunker.py @@ -0,0 +1,86 @@ +""" +Chunker — probes a media file and yields time-based Chunk objects. + +Demonstrates: +- Function parameters and defaults (Interview Topic 1) +- List comprehensions and efficient iteration / generators (Interview Topic 3) +""" + +import math +import os +from typing import Generator + +from core.ffmpeg.probe import probe_file + +from .exceptions import ChunkReadError +from .models import Chunk + + +class Chunker: + """ + Splits a media file into time-based chunks via a generator. + + Uses FFmpeg probe to get duration, then yields Chunk objects + representing time segments (no data read — extraction happens in the processor). + + Args: + file_path: Path to the source media file + chunk_duration: Duration of each chunk in seconds (default: 10.0) + """ + + def __init__(self, file_path: str, chunk_duration: float = 10.0): + if not os.path.isfile(file_path): + raise ChunkReadError(f"File not found: {file_path}") + if chunk_duration <= 0: + raise ValueError("chunk_duration must be positive") + + self.file_path = file_path + self.chunk_duration = chunk_duration + self.file_size = os.path.getsize(file_path) + self.source_duration = self._probe_duration() + + def _probe_duration(self) -> float: + """Get source file duration via FFmpeg probe.""" + try: + result = probe_file(self.file_path) + if result.duration is None or result.duration <= 0: + raise ChunkReadError( + f"Cannot determine duration for {self.file_path}" + ) + return result.duration + except ChunkReadError: + raise + except Exception as e: + raise ChunkReadError( + f"Failed to probe {self.file_path}: {e}" + ) from e + + @property + def expected_chunks(self) -> int: + """Calculate expected number of chunks (last chunk may be shorter).""" + if self.source_duration <= 0: + return 0 + return math.ceil(self.source_duration / self.chunk_duration) + + def chunks(self) -> Generator[Chunk, None, None]: + """ + Yield Chunk objects representing time segments of the source file. + + Generator-based: chunks are yielded on demand. + Each chunk defines a time range — actual extraction is done by the processor. + """ + total = self.expected_chunks + for sequence in range(total): + start_time = sequence * self.chunk_duration + end_time = min( + start_time + self.chunk_duration, self.source_duration + ) + duration = end_time - start_time + + yield Chunk( + sequence=sequence, + start_time=start_time, + end_time=end_time, + source_path=self.file_path, + duration=duration, + ) diff --git a/core/chunker/collector.py b/core/chunker/collector.py new file mode 100644 index 0000000..4e4b5fd --- /dev/null +++ b/core/chunker/collector.py @@ -0,0 +1,98 @@ +""" +ResultCollector — reassembles chunk results in sequence order using a min-heap. + +Demonstrates: +- Algorithms and sorting (Interview Topic 6) — heapq for ordered reassembly +- Core data structures (Interview Topic 5) — heap, deque +""" + +import heapq +from collections import deque +from typing import List + +from .exceptions import ReassemblyError +from .models import ChunkResult + + +class ResultCollector: + """ + Receives ChunkResults out of order, emits them in sequence order. + + Uses a min-heap keyed on sequence number. Only emits a chunk when + all prior sequences have been accounted for. + + Args: + total_chunks: Expected total number of chunks + """ + + def __init__(self, total_chunks: int): + self.total_chunks = total_chunks + self._heap: List[tuple[int, ChunkResult]] = [] + self._next_sequence = 0 + self._emitted: List[ChunkResult] = [] + self._seen_sequences: set[int] = set() + # Sliding window for throughput calculation + self._recent_times: deque[float] = deque(maxlen=50) + + def add(self, result: ChunkResult) -> List[ChunkResult]: + """ + Add a result and return any newly emittable results in order. + + Args: + result: A ChunkResult (may arrive out of order) + + Returns: + List of results that can now be emitted in sequence order + (may be empty if we're still waiting for earlier sequences) + + Raises: + ReassemblyError: If a duplicate sequence is received + """ + if result.sequence in self._seen_sequences: + raise ReassemblyError( + f"Duplicate sequence number: {result.sequence}" + ) + self._seen_sequences.add(result.sequence) + + # Track processing time for throughput + if result.processing_time > 0: + self._recent_times.append(result.processing_time) + + # Push to min-heap + heapq.heappush(self._heap, (result.sequence, result)) + + # Emit all consecutive results starting from _next_sequence + newly_emitted = [] + while self._heap and self._heap[0][0] == self._next_sequence: + _, emitted_result = heapq.heappop(self._heap) + self._emitted.append(emitted_result) + newly_emitted.append(emitted_result) + self._next_sequence += 1 + + return newly_emitted + + @property + def is_complete(self) -> bool: + """True if all expected chunks have been emitted in order.""" + return self._next_sequence == self.total_chunks + + @property + def buffered_count(self) -> int: + """Number of results waiting in the heap (arrived out of order).""" + return len(self._heap) + + @property + def emitted_count(self) -> int: + """Number of results emitted in sequence order.""" + return len(self._emitted) + + @property + def avg_processing_time(self) -> float: + """Average processing time from recent results (sliding window).""" + if not self._recent_times: + return 0.0 + return sum(self._recent_times) / len(self._recent_times) + + def get_ordered_results(self) -> List[ChunkResult]: + """Get all emitted results in sequence order.""" + return list(self._emitted) diff --git a/core/chunker/exceptions.py b/core/chunker/exceptions.py new file mode 100644 index 0000000..2426152 --- /dev/null +++ b/core/chunker/exceptions.py @@ -0,0 +1,64 @@ +""" +Chunker exception hierarchy. + +Demonstrates: Managing exceptions and writing resilient code (Interview Topic 7). +""" + + +class PipelineError(Exception): + """Base exception for all chunker pipeline errors.""" + pass + + +class ChunkError(PipelineError): + """Errors related to chunk creation or validation.""" + pass + + +class ChunkReadError(ChunkError): + """Failed to read chunk data from source file.""" + pass + + +class ChunkChecksumError(ChunkError): + """Chunk data integrity validation failed.""" + + def __init__(self, sequence: int, expected: str, actual: str): + self.sequence = sequence + self.expected = expected + self.actual = actual + super().__init__( + f"Chunk {sequence}: checksum mismatch " + f"(expected={expected}, actual={actual})" + ) + + +class ProcessingError(PipelineError): + """Errors during chunk processing by workers.""" + pass + + +class ProcessorTimeoutError(ProcessingError): + """Processor exceeded allowed time for a chunk.""" + + def __init__(self, sequence: int, timeout: float): + self.sequence = sequence + self.timeout = timeout + super().__init__(f"Chunk {sequence}: processor timed out after {timeout}s") + + +class ProcessorFailureError(ProcessingError): + """Processor failed to process a chunk after all retries.""" + + def __init__(self, sequence: int, retries: int, original_error: Exception): + self.sequence = sequence + self.retries = retries + self.original_error = original_error + super().__init__( + f"Chunk {sequence}: failed after {retries} retries — {original_error}" + ) + + +class ReassemblyError(PipelineError): + """Errors during result collection and ordering.""" + pass diff --git a/core/chunker/models.py b/core/chunker/models.py new file mode 100644 index 0000000..d2f6a7d --- /dev/null +++ b/core/chunker/models.py @@ -0,0 +1,54 @@ +""" +Internal data models for the chunker pipeline. + +These are pipeline-internal dataclasses, not schema models. +Schema-level ChunkJob is in core/schema/models/jobs.py. + +Demonstrates: Core data structures (Interview Topic 5). +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class Chunk: + """A time-based segment of the source media file.""" + + sequence: int + start_time: float # seconds + end_time: float # seconds + source_path: str # path to source file + duration: float # end_time - start_time + checksum: str = "" # computed after extraction + + +@dataclass +class ChunkResult: + """Result of processing a single chunk.""" + + sequence: int + success: bool + checksum_valid: bool = True + processing_time: float = 0.0 + error: Optional[str] = None + retries: int = 0 + worker_id: Optional[str] = None + output_file: Optional[str] = None + + +@dataclass +class PipelineResult: + """Aggregate result of the entire pipeline run.""" + + total_chunks: int = 0 + processed: int = 0 + failed: int = 0 + retries: int = 0 + elapsed_time: float = 0.0 + throughput_mbps: float = 0.0 + worker_stats: Dict[str, Any] = field(default_factory=dict) + errors: List[str] = field(default_factory=list) + chunks_in_order: bool = True + output_dir: Optional[str] = None + chunk_files: List[str] = field(default_factory=list) diff --git a/core/chunker/pipeline.py b/core/chunker/pipeline.py new file mode 100644 index 0000000..9bac5b8 --- /dev/null +++ b/core/chunker/pipeline.py @@ -0,0 +1,244 @@ +""" +Pipeline — orchestrates the entire chunker pipeline. + +Wires: Chunker → ChunkQueue → WorkerPool → ResultCollector → PipelineResult + +Demonstrates: +- Function parameters and defaults (Interview Topic 1) — configurable pipeline +- Concurrency (Interview Topic 2) — producer thread + worker pool +- OOP design (Interview Topic 4) — composition of pipeline components +- Exception handling (Interview Topic 7) — graceful error propagation +""" + +import json +import logging +import threading +import time +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +from .chunker import Chunker +from .collector import ResultCollector +from .exceptions import PipelineError +from .models import PipelineResult +from .pool import WorkerPool +from .queue import ChunkQueue + +logger = logging.getLogger(__name__) + + +class Pipeline: + """ + Orchestrates the chunk processing pipeline. + + The pipeline runs in three stages: + 1. Producer thread: Chunker probes file → pushes time-based chunks to ChunkQueue + 2. Worker pool: N workers pull from queue → extract mp4 segments → emit results + 3. Collector: ResultCollector reassembles results in sequence order + + Args: + source: Path to the source media file + chunk_duration: Duration of each chunk in seconds (default: 10.0) + num_workers: Number of concurrent worker threads (default: 4) + max_retries: Max retry attempts per chunk (default: 3) + processor_type: Processor to use — "ffmpeg", "checksum", "simulated_decode", "composite" + queue_size: Max chunks buffered in queue (default: 10) + event_callback: Optional callback for real-time events + output_dir: Directory for output chunk files (required for "ffmpeg" processor) + """ + + def __init__( + self, + source: str, + chunk_duration: float = 10.0, + num_workers: int = 4, + max_retries: int = 3, + processor_type: str = "checksum", + queue_size: int = 10, + event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + output_dir: Optional[str] = None, + ): + self.source = source + self.chunk_duration = chunk_duration + self.num_workers = num_workers + self.max_retries = max_retries + self.processor_type = processor_type + self.queue_size = queue_size + self.event_callback = event_callback + self.output_dir = output_dir + + def _emit(self, event_type: str, data: Dict[str, Any]) -> None: + """Emit an event if callback is registered.""" + if self.event_callback: + self.event_callback(event_type, data) + + def _produce_chunks( + self, chunker: Chunker, chunk_queue: ChunkQueue + ) -> None: + """Producer thread: probe file and enqueue time-based chunks.""" + try: + for chunk in chunker.chunks(): + chunk_queue.put(chunk, timeout=30.0) + self._emit("chunk_queued", { + "sequence": chunk.sequence, + "start_time": chunk.start_time, + "end_time": chunk.end_time, + "duration": chunk.duration, + "queue_size": chunk_queue.qsize(), + }) + except Exception as e: + logger.error(f"Producer error: {e}") + self._emit("producer_error", {"error": str(e)}) + finally: + chunk_queue.close() + + def _write_manifest( + self, result: PipelineResult, source_duration: float + ) -> None: + """Write manifest.json to output_dir with segment metadata.""" + if not self.output_dir: + return + + manifest = { + "source": self.source, + "source_duration": source_duration, + "chunk_duration": self.chunk_duration, + "total_chunks": result.total_chunks, + "processed": result.processed, + "failed": result.failed, + "elapsed_time": result.elapsed_time, + "throughput_mbps": result.throughput_mbps, + "segments": [ + { + "sequence": i, + "file": f"chunk_{i:04d}.mp4", + "start": i * self.chunk_duration, + "end": min( + (i + 1) * self.chunk_duration, source_duration + ), + } + for i in range(result.total_chunks) + if i < result.total_chunks + ], + } + + manifest_path = Path(self.output_dir) / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2)) + logger.info(f"Manifest written to {manifest_path}") + + def run(self) -> PipelineResult: + """ + Execute the full pipeline. + + Returns: + PipelineResult with aggregate stats + + Raises: + PipelineError: If the pipeline fails catastrophically + """ + start_time = time.monotonic() + self._emit("pipeline_start", { + "source": self.source, + "chunk_duration": self.chunk_duration, + "num_workers": self.num_workers, + "processor_type": self.processor_type, + }) + + try: + # Stage 1: Set up chunker (probes file for duration) + chunker = Chunker(self.source, self.chunk_duration) + total_chunks = chunker.expected_chunks + + if total_chunks == 0: + self._emit("pipeline_complete", {"total_chunks": 0}) + return PipelineResult(chunks_in_order=True) + + self._emit("pipeline_info", { + "file_size": chunker.file_size, + "source_duration": chunker.source_duration, + "total_chunks": total_chunks, + }) + + # Stage 2: Set up queue and worker pool + chunk_queue = ChunkQueue(maxsize=self.queue_size) + pool = WorkerPool( + num_workers=self.num_workers, + chunk_queue=chunk_queue, + processor_type=self.processor_type, + max_retries=self.max_retries, + event_callback=self.event_callback, + output_dir=self.output_dir, + ) + + # Stage 3: Start workers, then produce chunks + pool.start() + + producer = threading.Thread( + target=self._produce_chunks, + args=(chunker, chunk_queue), + name="chunk-producer", + daemon=True, + ) + producer.start() + + # Stage 4: Wait for all workers to finish + all_results = pool.wait() + producer.join(timeout=5.0) + + # Stage 5: Collect results in order + collector = ResultCollector(total_chunks) + for r in all_results: + collector.add(r) + self._emit("chunk_collected", { + "sequence": r.sequence, + "success": r.success, + "buffered": collector.buffered_count, + "emitted": collector.emitted_count, + }) + + # Build result + elapsed = time.monotonic() - start_time + file_size_mb = chunker.file_size / (1024 * 1024) + throughput = file_size_mb / elapsed if elapsed > 0 else 0.0 + + failed_results = [r for r in all_results if not r.success] + total_retries = sum(r.retries for r in all_results) + chunk_files = [ + r.output_file for r in all_results + if r.success and r.output_file + ] + + result = PipelineResult( + total_chunks=total_chunks, + processed=len(all_results), + failed=len(failed_results), + retries=total_retries, + elapsed_time=elapsed, + throughput_mbps=throughput, + worker_stats=pool.get_worker_stats(), + errors=[r.error for r in failed_results if r.error], + chunks_in_order=collector.is_complete, + output_dir=self.output_dir, + chunk_files=chunk_files, + ) + + # Write manifest if output_dir is set + self._write_manifest(result, chunker.source_duration) + + pool.shutdown() + + self._emit("pipeline_complete", { + "total_chunks": result.total_chunks, + "processed": result.processed, + "failed": result.failed, + "elapsed": result.elapsed_time, + "throughput_mbps": result.throughput_mbps, + }) + + return result + + except PipelineError: + raise + except Exception as e: + self._emit("pipeline_error", {"error": str(e)}) + raise PipelineError(f"Pipeline failed: {e}") from e diff --git a/core/chunker/pool.py b/core/chunker/pool.py new file mode 100644 index 0000000..bc86d04 --- /dev/null +++ b/core/chunker/pool.py @@ -0,0 +1,125 @@ +""" +WorkerPool — manages N worker threads via ThreadPoolExecutor. + +Demonstrates: Python concurrency — threading (Interview Topic 2). +""" + +import logging +import threading +from concurrent.futures import Future, ThreadPoolExecutor +from typing import Any, Callable, Dict, List, Optional + +from .models import ChunkResult +from .processor import ( + ChecksumProcessor, + CompositeProcessor, + FFmpegExtractProcessor, + Processor, + SimulatedDecodeProcessor, +) +from .queue import ChunkQueue +from .worker import Worker + +logger = logging.getLogger(__name__) + + +def create_processor( + processor_type: str = "checksum", + output_dir: Optional[str] = None, +) -> Processor: + """Factory for processor instances.""" + if processor_type == "ffmpeg": + if not output_dir: + raise ValueError("output_dir required for ffmpeg processor") + return FFmpegExtractProcessor(output_dir=output_dir) + elif processor_type == "checksum": + return ChecksumProcessor() + elif processor_type == "simulated_decode": + return SimulatedDecodeProcessor() + elif processor_type == "composite": + return CompositeProcessor([ + ChecksumProcessor(), + SimulatedDecodeProcessor(ms_per_second=50.0), + ]) + else: + raise ValueError(f"Unknown processor type: {processor_type}") + + +class WorkerPool: + """ + Manages N worker threads that process chunks concurrently. + + Args: + num_workers: Number of concurrent worker threads (default: 4) + chunk_queue: Shared queue to pull chunks from + processor_type: Type of processor for each worker (default: "checksum") + max_retries: Max retry attempts per chunk (default: 3) + event_callback: Optional callback for real-time events + """ + + def __init__( + self, + num_workers: int = 4, + chunk_queue: Optional[ChunkQueue] = None, + processor_type: str = "checksum", + max_retries: int = 3, + event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + output_dir: Optional[str] = None, + ): + self.num_workers = num_workers + self.chunk_queue = chunk_queue or ChunkQueue() + self.processor_type = processor_type + self.max_retries = max_retries + self.event_callback = event_callback + self.output_dir = output_dir + self.shutdown_event = threading.Event() + self._executor: Optional[ThreadPoolExecutor] = None + self._futures: List[Future] = [] + self._workers: List[Worker] = [] + + def start(self) -> None: + """Start all worker threads.""" + self._executor = ThreadPoolExecutor( + max_workers=self.num_workers, + thread_name_prefix="chunk-worker", + ) + + for i in range(self.num_workers): + worker = Worker( + worker_id=f"worker-{i}", + chunk_queue=self.chunk_queue, + processor=create_processor(self.processor_type, output_dir=self.output_dir), + max_retries=self.max_retries, + event_callback=self.event_callback, + ) + self._workers.append(worker) + future = self._executor.submit(worker.run) + self._futures.append(future) + + logger.info(f"WorkerPool started with {self.num_workers} workers") + + def wait(self) -> List[ChunkResult]: + """Wait for all workers to finish and collect results.""" + all_results = [] + for future in self._futures: + results = future.result() + all_results.extend(results) + return all_results + + def shutdown(self) -> None: + """Signal shutdown and cleanup.""" + self.shutdown_event.set() + self.chunk_queue.close() + if self._executor: + self._executor.shutdown(wait=True) + + def get_worker_stats(self) -> Dict[str, Any]: + """Get per-worker statistics.""" + return { + w.worker_id: { + "processed": w.processed_count, + "errors": w.error_count, + "retries": w.retry_count, + } + for w in self._workers + } diff --git a/core/chunker/processor.py b/core/chunker/processor.py new file mode 100644 index 0000000..dd5d772 --- /dev/null +++ b/core/chunker/processor.py @@ -0,0 +1,173 @@ +""" +Processor ABC and concrete implementations. + +Demonstrates: OOP design principles — ABC, inheritance, composition (Interview Topic 4). +""" + +import hashlib +import time +from abc import ABC, abstractmethod +from pathlib import Path +from typing import List + +from .exceptions import ChunkChecksumError +from .models import Chunk, ChunkResult + + +class Processor(ABC): + """ + Abstract base class for chunk processors. + + Each processor defines how a single chunk is processed. + The Worker calls processor.process(chunk) and handles retries. + """ + + @abstractmethod + def process(self, chunk: Chunk) -> ChunkResult: + """Process a single chunk and return the result.""" + pass + + +class FFmpegExtractProcessor(Processor): + """ + Extracts a time segment from the source file using FFmpeg stream copy. + + Produces a playable mp4 file per chunk — no re-encoding. + + Args: + output_dir: Directory to write chunk mp4 files + """ + + def __init__(self, output_dir: str): + self.output_dir = output_dir + Path(output_dir).mkdir(parents=True, exist_ok=True) + + def process(self, chunk: Chunk) -> ChunkResult: + from core.ffmpeg.transcode import TranscodeConfig, transcode + + start = time.monotonic() + + output_file = str( + Path(self.output_dir) / f"chunk_{chunk.sequence:04d}.mp4" + ) + + config = TranscodeConfig( + input_path=chunk.source_path, + output_path=output_file, + video_codec="copy", + audio_codec="copy", + trim_start=chunk.start_time, + trim_end=chunk.end_time, + ) + + transcode(config) + + # Compute checksum of output file + md5 = hashlib.md5() + with open(output_file, "rb") as f: + for block in iter(lambda: f.read(8192), b""): + md5.update(block) + checksum = md5.hexdigest() + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=True, + processing_time=elapsed, + output_file=output_file, + ) + + +class ChecksumProcessor(Processor): + """ + Validates chunk metadata consistency. + + For time-based chunks, verifies the time range is valid. + Raises ChunkChecksumError on invalid ranges. + """ + + def process(self, chunk: Chunk) -> ChunkResult: + start = time.monotonic() + + valid = chunk.duration > 0 and chunk.end_time > chunk.start_time + + if not valid: + raise ChunkChecksumError( + sequence=chunk.sequence, + expected="valid time range", + actual=f"{chunk.start_time}-{chunk.end_time}", + ) + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=True, + processing_time=elapsed, + ) + + +class SimulatedDecodeProcessor(Processor): + """ + Simulates decode work by sleeping proportional to chunk duration. + + Useful for demonstrating concurrency behavior without real FFmpeg. + + Args: + ms_per_second: Milliseconds of simulated work per second of chunk duration (default: 100) + """ + + def __init__(self, ms_per_second: float = 100.0): + self.ms_per_second = ms_per_second + + def process(self, chunk: Chunk) -> ChunkResult: + start = time.monotonic() + + sleep_time = (self.ms_per_second * chunk.duration) / 1000.0 + time.sleep(sleep_time) + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=True, + processing_time=elapsed, + ) + + +class CompositeProcessor(Processor): + """ + Chains multiple processors — runs each in sequence on the same chunk. + + Demonstrates OOP composition pattern. + + Args: + processors: List of processors to chain + """ + + def __init__(self, processors: List[Processor]): + if not processors: + raise ValueError("CompositeProcessor requires at least one processor") + self.processors = processors + + def process(self, chunk: Chunk) -> ChunkResult: + start = time.monotonic() + last_result = None + + for proc in self.processors: + last_result = proc.process(chunk) + if not last_result.success: + return last_result + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=last_result.checksum_valid if last_result else True, + processing_time=elapsed, + ) diff --git a/core/chunker/queue.py b/core/chunker/queue.py new file mode 100644 index 0000000..191a219 --- /dev/null +++ b/core/chunker/queue.py @@ -0,0 +1,76 @@ +""" +ChunkQueue — bounded, thread-safe queue with sentinel-based shutdown. + +Demonstrates: Core data structures — queue.Queue (Interview Topic 5). +""" + +import queue +from typing import Optional + +from .models import Chunk + +# Sentinel value to signal workers to stop +_SENTINEL = object() + + +class ChunkQueue: + """ + Thread-safe bounded queue for chunks. + + Provides backpressure: producers block when the queue is full, + preventing unbounded memory usage. + + Args: + maxsize: Maximum number of chunks in the queue (default: 10) + """ + + def __init__(self, maxsize: int = 10): + self._queue: queue.Queue = queue.Queue(maxsize=maxsize) + self._closed = False + self.maxsize = maxsize + + def put(self, chunk: Chunk, timeout: Optional[float] = None) -> None: + """ + Add a chunk to the queue. Blocks if full (backpressure). + + Args: + chunk: The chunk to enqueue + timeout: Max seconds to wait (None = block forever) + + Raises: + queue.Full: If timeout expires while queue is full + """ + self._queue.put(chunk, timeout=timeout) + + def get(self, timeout: Optional[float] = None) -> Optional[Chunk]: + """ + Get next chunk from queue. Returns None if queue is closed. + + Args: + timeout: Max seconds to wait (None = block forever) + + Returns: + Chunk or None (if sentinel received, meaning queue is closed) + + Raises: + queue.Empty: If timeout expires while queue is empty + """ + item = self._queue.get(timeout=timeout) + if item is _SENTINEL: + # Re-put sentinel so other workers also see it + self._queue.put(_SENTINEL) + return None + return item + + def close(self) -> None: + """Signal all consumers to stop by inserting a sentinel.""" + self._closed = True + self._queue.put(_SENTINEL) + + @property + def is_closed(self) -> bool: + return self._closed + + def qsize(self) -> int: + """Current number of items in the queue (approximate).""" + return self._queue.qsize() diff --git a/core/chunker/worker.py b/core/chunker/worker.py new file mode 100644 index 0000000..2fcc550 --- /dev/null +++ b/core/chunker/worker.py @@ -0,0 +1,141 @@ +""" +Worker — pulls chunks from queue, processes with retry logic. + +Demonstrates: +- Exception handling and resilient code (Interview Topic 7) +- Concurrency (Interview Topic 2) — workers run in thread pool +""" + +import logging +import queue +import time +from typing import Any, Callable, Dict, Optional + +from .exceptions import ProcessorFailureError +from .models import Chunk, ChunkResult +from .processor import Processor +from .queue import ChunkQueue + +logger = logging.getLogger(__name__) + + +class Worker: + """ + Processes chunks from a queue with retry and exponential backoff. + + Args: + worker_id: Identifier for this worker (e.g. "worker-0") + chunk_queue: Source queue to pull chunks from + processor: Processor instance to use + max_retries: Maximum retry attempts per chunk (default: 3) + event_callback: Optional callback for real-time status updates + """ + + def __init__( + self, + worker_id: str, + chunk_queue: ChunkQueue, + processor: Processor, + max_retries: int = 3, + event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + ): + self.worker_id = worker_id + self.chunk_queue = chunk_queue + self.processor = processor + self.max_retries = max_retries + self.event_callback = event_callback + self.processed_count = 0 + self.error_count = 0 + self.retry_count = 0 + + def _emit(self, event_type: str, data: Dict[str, Any]) -> None: + """Emit an event if callback is registered.""" + if self.event_callback: + self.event_callback(event_type, {"worker_id": self.worker_id, **data}) + + def _process_with_retry(self, chunk: Chunk) -> ChunkResult: + """ + Process a chunk with exponential backoff retry. + + Retry delays: 0.1s, 0.2s, 0.4s, ... (doubles each attempt) + """ + last_error = None + + for attempt in range(self.max_retries + 1): + try: + if attempt > 0: + backoff = 0.1 * (2 ** (attempt - 1)) + self._emit("chunk_retry", { + "sequence": chunk.sequence, + "attempt": attempt, + "backoff": backoff, + }) + time.sleep(backoff) + self.retry_count += 1 + + result = self.processor.process(chunk) + result.retries = attempt + result.worker_id = self.worker_id + return result + + except Exception as e: + last_error = e + logger.warning( + f"{self.worker_id}: chunk {chunk.sequence} " + f"attempt {attempt + 1}/{self.max_retries + 1} failed: {e}" + ) + + # All retries exhausted + self.error_count += 1 + self._emit("chunk_error", { + "sequence": chunk.sequence, + "error": str(last_error), + "retries": self.max_retries, + }) + + return ChunkResult( + sequence=chunk.sequence, + success=False, + processing_time=0.0, + error=str(last_error), + retries=self.max_retries, + worker_id=self.worker_id, + ) + + def run(self) -> list[ChunkResult]: + """ + Main worker loop — pull chunks and process until queue is closed. + + Returns: + List of ChunkResults processed by this worker + """ + results = [] + self._emit("worker_status", {"state": "idle"}) + + while True: + try: + chunk = self.chunk_queue.get(timeout=1.0) + except queue.Empty: + continue + + if chunk is None: # Sentinel received + break + + self._emit("chunk_processing", { + "sequence": chunk.sequence, + "state": "processing", + }) + + result = self._process_with_retry(chunk) + results.append(result) + self.processed_count += 1 + + self._emit("chunk_done", { + "sequence": chunk.sequence, + "success": result.success, + "processing_time": result.processing_time, + "retries": result.retries, + }) + + self._emit("worker_status", {"state": "stopped"}) + return results diff --git a/core/jobs/__init__.py b/core/jobs/__init__.py new file mode 100644 index 0000000..8827db9 --- /dev/null +++ b/core/jobs/__init__.py @@ -0,0 +1,15 @@ +""" +MPR Jobs Module + +Provides executor abstraction and task dispatch for job processing. +""" + +from .executor import Executor, LocalExecutor, get_executor +from .task import run_job + +__all__ = [ + "Executor", + "LocalExecutor", + "get_executor", + "run_job", +] diff --git a/core/task/executor.py b/core/jobs/executor.py similarity index 53% rename from core/task/executor.py rename to core/jobs/executor.py index aadf783..ef1e6cd 100644 --- a/core/task/executor.py +++ b/core/jobs/executor.py @@ -1,17 +1,16 @@ """ Executor abstraction for job processing. -Supports different backends: -- LocalExecutor: FFmpeg via Celery (default) -- LambdaExecutor: AWS Lambda (future) +Determines WHERE jobs run: +- LocalExecutor: delegates to registered Handler (default) +- LambdaExecutor: AWS Step Functions +- GCPExecutor: Google Cloud Run Jobs """ import os from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Optional -from core.ffmpeg.transcode import TranscodeConfig, transcode - # Configuration from environment MPR_EXECUTOR = os.environ.get("MPR_EXECUTOR", "local") @@ -22,26 +21,18 @@ class Executor(ABC): @abstractmethod def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: """ - Execute a transcode/trim job. + Execute a job. Args: + job_type: Type of job ("transcode", "chunk", etc.) job_id: Unique job identifier - source_path: Path to source file - output_path: Path for output file - preset: Transcode preset dict (optional, None = trim only) - trim_start: Trim start time in seconds (optional) - trim_end: Trim end time in seconds (optional) - duration: Source duration in seconds (for progress calculation) + payload: Job-type-specific configuration dict progress_callback: Called with (percent, details_dict) Returns: @@ -51,62 +42,25 @@ class Executor(ABC): class LocalExecutor(Executor): - """Execute jobs locally using FFmpeg.""" + """Execute jobs locally using registered handlers.""" def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: - """Execute job using local FFmpeg.""" + """Execute job using the appropriate local handler.""" + from .registry import get_handler - # Build config from preset or use stream copy for trim-only - if preset: - config = TranscodeConfig( - input_path=source_path, - output_path=output_path, - video_codec=preset.get("video_codec", "libx264"), - video_bitrate=preset.get("video_bitrate"), - video_crf=preset.get("video_crf"), - video_preset=preset.get("video_preset"), - resolution=preset.get("resolution"), - framerate=preset.get("framerate"), - audio_codec=preset.get("audio_codec", "aac"), - audio_bitrate=preset.get("audio_bitrate"), - audio_channels=preset.get("audio_channels"), - audio_samplerate=preset.get("audio_samplerate"), - container=preset.get("container", "mp4"), - extra_args=preset.get("extra_args", []), - trim_start=trim_start, - trim_end=trim_end, - ) - else: - # Trim-only: stream copy - config = TranscodeConfig( - input_path=source_path, - output_path=output_path, - video_codec="copy", - audio_codec="copy", - trim_start=trim_start, - trim_end=trim_end, - ) - - # Wrapper to convert float percent to int - def wrapped_callback(percent: float, details: Dict[str, Any]) -> None: - if progress_callback: - progress_callback(int(percent), details) - - return transcode( - config, - duration=duration, - progress_callback=wrapped_callback if progress_callback else None, + handler = get_handler(job_type) + result = handler.process( + job_id=job_id, + payload=payload, + progress_callback=progress_callback, ) + return result.get("status") == "completed" class LambdaExecutor(Executor): @@ -123,26 +77,18 @@ class LambdaExecutor(Executor): def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: """Start a Step Functions execution for this job.""" import json - payload = { + sfn_payload = { + "job_type": job_type, "job_id": job_id, - "source_key": source_path, - "output_key": output_path, - "preset": preset, - "trim_start": trim_start, - "trim_end": trim_end, - "duration": duration, + **payload, "callback_url": self.callback_url, "api_key": self.callback_api_key, } @@ -150,10 +96,9 @@ class LambdaExecutor(Executor): response = self.sfn.start_execution( stateMachineArn=self.state_machine_arn, name=f"mpr-{job_id}", - input=json.dumps(payload), + input=json.dumps(sfn_payload), ) - # Store execution ARN on the job execution_arn = response["executionArn"] try: from core.db import update_job_fields @@ -179,13 +124,9 @@ class GCPExecutor(Executor): def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: """Trigger a Cloud Run Job execution for this job.""" @@ -193,14 +134,10 @@ class GCPExecutor(Executor): from google.cloud import run_v2 - payload = { + gcp_payload = { + "job_type": job_type, "job_id": job_id, - "source_key": source_path, - "output_key": output_path, - "preset": preset, - "trim_start": trim_start, - "trim_end": trim_end, - "duration": duration, + **payload, "callback_url": self.callback_url, "api_key": self.callback_api_key, } @@ -216,7 +153,8 @@ class GCPExecutor(Executor): run_v2.RunJobRequest.Overrides.ContainerOverride( env=[ run_v2.EnvVar( - name="MPR_JOB_PAYLOAD", value=json.dumps(payload) + name="MPR_JOB_PAYLOAD", + value=json.dumps(gcp_payload), ) ] ) diff --git a/core/task/gcp_handler.py b/core/jobs/gcp_handler.py similarity index 100% rename from core/task/gcp_handler.py rename to core/jobs/gcp_handler.py diff --git a/core/jobs/handlers/__init__.py b/core/jobs/handlers/__init__.py new file mode 100644 index 0000000..8ac89c6 --- /dev/null +++ b/core/jobs/handlers/__init__.py @@ -0,0 +1,5 @@ +"""Job handlers — type-specific execution logic.""" + +from .base import Handler + +__all__ = ["Handler"] diff --git a/core/jobs/handlers/base.py b/core/jobs/handlers/base.py new file mode 100644 index 0000000..f47328f --- /dev/null +++ b/core/jobs/handlers/base.py @@ -0,0 +1,33 @@ +""" +Base Handler ABC — defines the interface for job-type-specific execution logic. + +A Handler knows HOW to execute a specific kind of job (transcode, chunk, etc.). +The Executor decides WHERE to run it (local, Lambda, GCP). +""" + +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Optional + + +class Handler(ABC): + """Abstract base class for job handlers.""" + + @abstractmethod + def process( + self, + job_id: str, + payload: Dict[str, Any], + progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + """ + Execute job-specific logic. + + Args: + job_id: Unique job identifier + payload: Job-type-specific configuration + progress_callback: Called with (percent, details_dict) + + Returns: + Result dict with at least {"status": "completed"} or raises + """ + pass diff --git a/core/jobs/handlers/chunk.py b/core/jobs/handlers/chunk.py new file mode 100644 index 0000000..7a06a11 --- /dev/null +++ b/core/jobs/handlers/chunk.py @@ -0,0 +1,119 @@ +""" +ChunkHandler — job handler that wraps the chunker Pipeline. + +Downloads source from S3/MinIO, runs FFmpeg chunking pipeline, +uploads mp4 segments + manifest back to S3/MinIO. +""" + +import logging +import os +import shutil +import tempfile +from typing import Any, Callable, Dict, Optional + +from core.chunker import Pipeline +from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file + +from .base import Handler + +logger = logging.getLogger(__name__) + + +class ChunkHandler(Handler): + """ + Handles chunk processing jobs by delegating to the chunker Pipeline. + + Expected payload keys: + source_key: str — S3 key of the source file in BUCKET_IN + chunk_duration: float — seconds per chunk (default: 10.0) + num_workers: int — concurrent workers (default: 4) + max_retries: int — retries per chunk (default: 3) + processor_type: str — "ffmpeg", "checksum", "simulated_decode", "composite" + queue_size: int — max queue depth (default: 10) + """ + + def process( + self, + job_id: str, + payload: Dict[str, Any], + progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + source_key = payload["source_key"] + processor_type = payload.get("processor_type", "ffmpeg") + + logger.info(f"ChunkHandler starting job {job_id}: {source_key}") + + # Download source from S3/MinIO + tmp_source = download_to_temp(BUCKET_IN, source_key) + + # Create temp output directory for chunks + tmp_output_dir = tempfile.mkdtemp(prefix=f"chunks-{job_id}-") + + try: + def event_bridge(event_type: str, data: Dict[str, Any]) -> None: + """Bridge pipeline events to the job progress callback.""" + if progress_callback and event_type == "pipeline_complete": + progress_callback(100, data) + elif progress_callback and event_type == "chunk_done": + total = data.get("total_chunks", 1) + if total > 0: + pct = min(int((data.get("sequence", 0) + 1) / total * 100), 99) + progress_callback(pct, data) + + pipeline = Pipeline( + source=tmp_source, + chunk_duration=payload.get("chunk_duration", 10.0), + num_workers=payload.get("num_workers", 4), + max_retries=payload.get("max_retries", 3), + processor_type=processor_type, + queue_size=payload.get("queue_size", 10), + event_callback=event_bridge, + output_dir=tmp_output_dir if processor_type == "ffmpeg" else None, + ) + + result = pipeline.run() + + # Upload chunks + manifest to S3/MinIO + output_prefix = f"chunks/{job_id}" + uploaded_files = [] + + for chunk_file in result.chunk_files: + filename = os.path.basename(chunk_file) + output_key = f"{output_prefix}/{filename}" + upload_file(chunk_file, BUCKET_OUT, output_key) + uploaded_files.append(output_key) + logger.info(f"Uploaded {output_key}") + + # Upload manifest + manifest_path = os.path.join(tmp_output_dir, "manifest.json") + if os.path.exists(manifest_path): + manifest_key = f"{output_prefix}/manifest.json" + upload_file(manifest_path, BUCKET_OUT, manifest_key) + uploaded_files.append(manifest_key) + logger.info(f"Uploaded {manifest_key}") + + return { + "status": "completed" if result.failed == 0 else "completed_with_errors", + "total_chunks": result.total_chunks, + "processed": result.processed, + "failed": result.failed, + "retries": result.retries, + "elapsed_time": result.elapsed_time, + "throughput_mbps": result.throughput_mbps, + "worker_stats": result.worker_stats, + "errors": result.errors, + "chunks_in_order": result.chunks_in_order, + "output_prefix": output_prefix, + "uploaded_files": uploaded_files, + } + + finally: + # Cleanup temp files + try: + os.unlink(tmp_source) + except OSError: + pass + try: + shutil.rmtree(tmp_output_dir, ignore_errors=True) + except OSError: + pass diff --git a/core/jobs/handlers/transcode.py b/core/jobs/handlers/transcode.py new file mode 100644 index 0000000..6371e2f --- /dev/null +++ b/core/jobs/handlers/transcode.py @@ -0,0 +1,104 @@ +""" +TranscodeHandler — executes transcode/trim jobs using FFmpeg. + +Extracted from the old tasks.py Celery task logic. +""" + +import logging +import os +import tempfile +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +from core.ffmpeg.transcode import TranscodeConfig, transcode +from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file + +from .base import Handler + +logger = logging.getLogger(__name__) + + +class TranscodeHandler(Handler): + """Handle transcode and trim jobs via FFmpeg.""" + + def process( + self, + job_id: str, + payload: Dict[str, Any], + progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + source_key = payload["source_key"] + output_key = payload["output_key"] + preset = payload.get("preset") + trim_start = payload.get("trim_start") + trim_end = payload.get("trim_end") + duration = payload.get("duration") + + logger.info(f"TranscodeHandler: {source_key} -> {output_key}") + + # Download source + tmp_source = download_to_temp(BUCKET_IN, source_key) + + ext = Path(output_key).suffix or ".mp4" + fd, tmp_output = tempfile.mkstemp(suffix=ext) + os.close(fd) + + try: + if preset: + config = TranscodeConfig( + input_path=tmp_source, + output_path=tmp_output, + video_codec=preset.get("video_codec", "libx264"), + video_bitrate=preset.get("video_bitrate"), + video_crf=preset.get("video_crf"), + video_preset=preset.get("video_preset"), + resolution=preset.get("resolution"), + framerate=preset.get("framerate"), + audio_codec=preset.get("audio_codec", "aac"), + audio_bitrate=preset.get("audio_bitrate"), + audio_channels=preset.get("audio_channels"), + audio_samplerate=preset.get("audio_samplerate"), + container=preset.get("container", "mp4"), + extra_args=preset.get("extra_args", []), + trim_start=trim_start, + trim_end=trim_end, + ) + else: + config = TranscodeConfig( + input_path=tmp_source, + output_path=tmp_output, + video_codec="copy", + audio_codec="copy", + trim_start=trim_start, + trim_end=trim_end, + ) + + def wrapped_callback(percent: float, details: Dict[str, Any]) -> None: + if progress_callback: + progress_callback(int(percent), details) + + success = transcode( + config, + duration=duration, + progress_callback=wrapped_callback if progress_callback else None, + ) + + if not success: + raise RuntimeError("Transcode returned False") + + # Upload result + logger.info(f"Uploading {output_key} to {BUCKET_OUT}") + upload_file(tmp_output, BUCKET_OUT, output_key) + + return { + "status": "completed", + "job_id": job_id, + "output_key": output_key, + } + + finally: + for f in [tmp_source, tmp_output]: + try: + os.unlink(f) + except OSError: + pass diff --git a/core/task/lambda_handler.py b/core/jobs/lambda_handler.py similarity index 100% rename from core/task/lambda_handler.py rename to core/jobs/lambda_handler.py diff --git a/core/jobs/registry.py b/core/jobs/registry.py new file mode 100644 index 0000000..6b9b4f7 --- /dev/null +++ b/core/jobs/registry.py @@ -0,0 +1,33 @@ +""" +Handler registry — maps job_type strings to Handler classes. +""" + +from typing import Dict, Type + +from .handlers.base import Handler + +_handlers: Dict[str, Type[Handler]] = {} + + +def register_handler(job_type: str, handler_class: Type[Handler]) -> None: + """Register a handler class for a job type.""" + _handlers[job_type] = handler_class + + +def get_handler(job_type: str) -> Handler: + """Get an instantiated handler for a job type.""" + if job_type not in _handlers: + raise ValueError(f"Unknown job type: {job_type}") + return _handlers[job_type]() + + +def _register_defaults() -> None: + """Register built-in handlers.""" + from .handlers.chunk import ChunkHandler + from .handlers.transcode import TranscodeHandler + + register_handler("transcode", TranscodeHandler) + register_handler("chunk", ChunkHandler) + + +_register_defaults() diff --git a/core/jobs/task.py b/core/jobs/task.py new file mode 100644 index 0000000..4c0a60d --- /dev/null +++ b/core/jobs/task.py @@ -0,0 +1,64 @@ +""" +Celery task for job processing. + +Generic dispatcher — routes to the appropriate handler based on job_type. +""" + +import logging +from typing import Any, Dict + +from celery import shared_task + +from core.rpc.server import update_job_progress + +logger = logging.getLogger(__name__) + + +@shared_task(bind=True, max_retries=3, default_retry_delay=60) +def run_job( + self, + job_type: str, + job_id: str, + payload: Dict[str, Any], +) -> Dict[str, Any]: + """ + Generic Celery task — dispatches to the registered handler for job_type. + """ + logger.info(f"Starting {job_type} job {job_id}") + + update_job_progress(job_id, progress=0, status="processing") + + def progress_callback(percent: int, details: Dict[str, Any]) -> None: + update_job_progress( + job_id, + progress=percent, + current_time=details.get("time", 0.0), + status="processing", + ) + + try: + from .registry import get_handler + + handler = get_handler(job_type) + result = handler.process( + job_id=job_id, + payload=payload, + progress_callback=progress_callback, + ) + + logger.info(f"Job {job_id} completed successfully") + update_job_progress(job_id, progress=100, status="completed") + return result + + except Exception as e: + logger.exception(f"Job {job_id} failed: {e}") + update_job_progress(job_id, progress=0, status="failed", error=str(e)) + + if self.request.retries < self.max_retries: + raise self.retry(exc=e) + + return { + "status": "failed", + "job_id": job_id, + "error": str(e), + } diff --git a/core/rpc/server.py b/core/rpc/server.py index f4ae778..d07ce61 100644 --- a/core/rpc/server.py +++ b/core/rpc/server.py @@ -59,17 +59,24 @@ class WorkerServicer(worker_pb2_grpc.WorkerServiceServicer): # Dispatch to Celery if available if self.celery_app: - from core.task.tasks import run_transcode_job + from core.jobs.task import run_job - task = run_transcode_job.delay( - job_id=job_id, - source_path=request.source_path, - output_path=request.output_path, - preset=preset, - trim_start=request.trim_start + payload = { + "source_key": request.source_path, + "output_key": request.output_path, + "preset": preset, + "trim_start": request.trim_start if request.HasField("trim_start") else None, - trim_end=request.trim_end if request.HasField("trim_end") else None, + "trim_end": request.trim_end + if request.HasField("trim_end") + else None, + } + + task = run_job.delay( + job_type="transcode", + job_id=job_id, + payload=payload, ) _active_jobs[job_id]["celery_task_id"] = task.id @@ -197,11 +204,14 @@ def update_job_progress( speed: float = 0.0, status: str = "processing", error: str = None, + **extra, ) -> None: """ Update job progress (called from worker tasks). Updates both the in-memory gRPC state and the Django database. + Extra kwargs are stored for chunker-specific fields (total_chunks, + processed_chunks, failed_chunks, throughput_mbps, etc.). """ if job_id in _active_jobs: _active_jobs[job_id].update( @@ -212,6 +222,7 @@ def update_job_progress( "speed": speed, "status": status, "error": error, + **extra, } ) diff --git a/core/schema/models/__init__.py b/core/schema/models/__init__.py index 188440f..b0b0dcc 100644 --- a/core/schema/models/__init__.py +++ b/core/schema/models/__init__.py @@ -23,12 +23,12 @@ from .grpc import ( ProgressUpdate, WorkerStatus, ) -from .jobs import JobStatus, TranscodeJob +from .jobs import ChunkJob, ChunkJobStatus, JobStatus, TranscodeJob from .media import AssetStatus, MediaAsset from .presets import BUILTIN_PRESETS, TranscodePreset # Core domain models - generates Django, Pydantic, TypeScript -DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob] +DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob, ChunkJob] # API request/response models - generates TypeScript only (no Django) # WorkerStatus from grpc.py is reused here @@ -42,7 +42,7 @@ API_MODELS = [ ] # Status enums - included in generated code -ENUMS = [AssetStatus, JobStatus] +ENUMS = [AssetStatus, JobStatus, ChunkJobStatus] # gRPC messages - generates Proto GRPC_MESSAGES = [ @@ -61,6 +61,7 @@ __all__ = [ "MediaAsset", "TranscodePreset", "TranscodeJob", + "ChunkJob", # API Models "CreateJobRequest", "UpdateAssetRequest", @@ -70,6 +71,7 @@ __all__ = [ # Enums "AssetStatus", "JobStatus", + "ChunkJobStatus", # gRPC "GRPC_SERVICE", "JobRequest", diff --git a/core/schema/models/jobs.py b/core/schema/models/jobs.py index 8a7f6f0..0957034 100644 --- a/core/schema/models/jobs.py +++ b/core/schema/models/jobs.py @@ -1,13 +1,14 @@ """ -TranscodeJob Schema Definition +Job Schema Definitions -Source of truth for job data model. +Source of truth for job data models. +TranscodeJob and ChunkJob share common lifecycle fields by convention. """ from dataclasses import dataclass, field from datetime import datetime from enum import Enum -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from uuid import UUID @@ -77,3 +78,56 @@ class TranscodeJob: return self.preset_id is None and ( self.trim_start is not None or self.trim_end is not None ) + + +class ChunkJobStatus(str, Enum): + """Status of a chunk pipeline job.""" + + PENDING = "pending" + CHUNKING = "chunking" + PROCESSING = "processing" + COLLECTING = "collecting" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass +class ChunkJob: + """ + A chunk pipeline job — splits a media file into chunks and processes them + through a concurrent worker pool. + """ + + id: UUID + + # Input + source_asset_id: UUID + + # Configuration + chunk_duration: float = 10.0 # seconds + num_workers: int = 4 + max_retries: int = 3 + processor_type: str = "ffmpeg" # "ffmpeg", "checksum", "simulated_decode", "composite" + + # Status & Progress + status: ChunkJobStatus = ChunkJobStatus.PENDING + progress: float = 0.0 # 0.0 to 100.0 + total_chunks: int = 0 + processed_chunks: int = 0 + failed_chunks: int = 0 + retry_count: int = 0 + error_message: Optional[str] = None + + # Result stats + throughput_mbps: Optional[float] = None + elapsed_seconds: Optional[float] = None + + # Worker tracking + celery_task_id: Optional[str] = None + priority: int = 0 # Lower = higher priority + + # Timestamps + created_at: Optional[datetime] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None diff --git a/core/task/__init__.py b/core/task/__init__.py deleted file mode 100644 index fea2bad..0000000 --- a/core/task/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -MPR Worker Module - -Provides executor abstraction and Celery tasks for job processing. -""" - -from .executor import Executor, LocalExecutor, get_executor -from .tasks import run_transcode_job - -__all__ = [ - "Executor", - "LocalExecutor", - "get_executor", - "run_transcode_job", -] diff --git a/core/task/tasks.py b/core/task/tasks.py deleted file mode 100644 index 5e0713a..0000000 --- a/core/task/tasks.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -Celery tasks for job processing. -""" - -import logging -import os -from typing import Any, Dict, Optional - -from celery import shared_task - -from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file -from core.rpc.server import update_job_progress -from core.task.executor import get_executor - -logger = logging.getLogger(__name__) - - -@shared_task(bind=True, queue="transcode", max_retries=3, default_retry_delay=60) -def run_transcode_job( - self, - job_id: str, - source_key: str, - output_key: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, -) -> Dict[str, Any]: - """ - Celery task to run a transcode/trim job. - - Downloads source from S3, runs FFmpeg, uploads result to S3. - """ - logger.info(f"Starting job {job_id}: {source_key} -> {output_key}") - - update_job_progress(job_id, progress=0, status="processing") - - # Download source from S3 to temp file - logger.info(f"Downloading {source_key} from {BUCKET_IN}") - tmp_source = download_to_temp(BUCKET_IN, source_key) - - # Create temp output path with same extension - import tempfile - from pathlib import Path - - ext = Path(output_key).suffix or ".mp4" - fd, tmp_output = tempfile.mkstemp(suffix=ext) - os.close(fd) - - def progress_callback(percent: int, details: Dict[str, Any]) -> None: - update_job_progress( - job_id, - progress=percent, - current_time=details.get("time", 0.0), - status="processing", - ) - - try: - executor = get_executor() - success = executor.run( - job_id=job_id, - source_path=tmp_source, - output_path=tmp_output, - preset=preset, - trim_start=trim_start, - trim_end=trim_end, - duration=duration, - progress_callback=progress_callback, - ) - - if success: - # Upload result to S3 - logger.info(f"Uploading {output_key} to {BUCKET_OUT}") - upload_file(tmp_output, BUCKET_OUT, output_key) - - logger.info(f"Job {job_id} completed successfully") - update_job_progress(job_id, progress=100, status="completed") - return { - "status": "completed", - "job_id": job_id, - "output_key": output_key, - } - else: - raise RuntimeError("Executor returned False") - - except Exception as e: - logger.exception(f"Job {job_id} failed: {e}") - update_job_progress(job_id, progress=0, status="failed", error=str(e)) - - if self.request.retries < self.max_retries: - raise self.retry(exc=e) - - return { - "status": "failed", - "job_id": job_id, - "error": str(e), - } - - finally: - # Clean up temp files - for f in [tmp_source, tmp_output]: - try: - os.unlink(f) - except OSError: - pass diff --git a/ctrl/Dockerfile b/ctrl/Dockerfile index 80e2152..247b054 100644 --- a/ctrl/Dockerfile +++ b/ctrl/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY . . +# No COPY . . — code is volume-mounted in dev (..:/app) +# This image only provides the Python runtime + dependencies CMD ["python", "admin/manage.py", "runserver", "0.0.0.0:8000"] diff --git a/ctrl/Dockerfile.worker b/ctrl/Dockerfile.worker index bbb7a49..e69ff5b 100644 --- a/ctrl/Dockerfile.worker +++ b/ctrl/Dockerfile.worker @@ -9,6 +9,7 @@ WORKDIR /app COPY requirements.txt requirements-worker.txt ./ RUN pip install --no-cache-dir -r requirements-worker.txt -COPY . . +# No COPY . . — code is volume-mounted in dev (..:/app) +# This image only provides Python runtime + FFmpeg + dependencies CMD ["celery", "-A", "admin.mpr", "worker", "--loglevel=info"] diff --git a/ctrl/docker-compose.yml b/ctrl/docker-compose.yml index 2c114c7..bfbf700 100644 --- a/ctrl/docker-compose.yml +++ b/ctrl/docker-compose.yml @@ -17,6 +17,20 @@ x-healthcheck-defaults: &healthcheck-defaults timeout: 5s retries: 5 +x-python-service: &python-service + build: + context: .. + dockerfile: ctrl/Dockerfile + volumes: + - ..:/app + environment: + <<: *common-env + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + services: # ============================================================================= # Infrastructure @@ -92,47 +106,25 @@ services: # ============================================================================= django: - build: - context: .. - dockerfile: ctrl/Dockerfile + <<: *python-service command: > bash -c "python admin/manage.py migrate && python admin/manage.py loadbuiltins || true && python admin/manage.py runserver 0.0.0.0:8701" ports: - "8701:8701" - environment: - <<: *common-env - volumes: - - ..:/app - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy fastapi: - build: - context: .. - dockerfile: ctrl/Dockerfile + <<: *python-service command: uvicorn core.api.main:app --host 0.0.0.0 --port 8702 --reload ports: - "8702:8702" environment: <<: *common-env DJANGO_ALLOW_ASYNC_UNSAFE: "true" - volumes: - - ..:/app - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy grpc: - build: - context: .. - dockerfile: ctrl/Dockerfile + <<: *python-service command: python -m core.rpc.server ports: - "50052:50051" @@ -140,13 +132,6 @@ services: <<: *common-env GRPC_PORT: 50051 GRPC_MAX_WORKERS: 10 - volumes: - - ..:/app - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy celery: build: diff --git a/ctrl/lambda/Dockerfile b/ctrl/lambda/Dockerfile index 52c0b82..2869686 100644 --- a/ctrl/lambda/Dockerfile +++ b/ctrl/lambda/Dockerfile @@ -14,8 +14,8 @@ COPY ctrl/lambda/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # Copy application code -COPY core/task/lambda_handler.py ${LAMBDA_TASK_ROOT}/core/task/lambda_handler.py -COPY core/task/__init__.py ${LAMBDA_TASK_ROOT}/core/task/__init__.py +COPY core/jobs/lambda_handler.py ${LAMBDA_TASK_ROOT}/core/jobs/lambda_handler.py +COPY core/jobs/__init__.py ${LAMBDA_TASK_ROOT}/core/jobs/__init__.py COPY core/ ${LAMBDA_TASK_ROOT}/core/ -CMD ["core.task.lambda_handler.handler"] +CMD ["core.jobs.lambda_handler.handler"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/chunker/__init__.py b/tests/chunker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/chunker/conftest.py b/tests/chunker/conftest.py new file mode 100644 index 0000000..1ddafe6 --- /dev/null +++ b/tests/chunker/conftest.py @@ -0,0 +1,76 @@ +""" +Shared fixtures for chunker tests. + +Demonstrates: TDD and unit testing best practices (Interview Topic 8) — fixtures, temp files. +""" + +import os +import tempfile + +import pytest + +from core.chunker.models import Chunk, ChunkResult + + +@pytest.fixture +def temp_file(): + """Create a temporary file with known content, cleaned up after test.""" + files = [] + + def _create(content: bytes = b"x" * 4096): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") + f.write(content) + f.close() + files.append(f.name) + return f.name + + yield _create + + for path in files: + if os.path.exists(path): + os.unlink(path) + + +@pytest.fixture +def sample_chunk(temp_file): + """Create a sample time-based Chunk with valid time range.""" + path = temp_file(b"x" * 1024) + return Chunk( + sequence=0, + start_time=0.0, + end_time=10.0, + source_path=path, + duration=10.0, + ) + + +@pytest.fixture +def make_chunk(temp_file): + """Factory fixture for creating time-based chunks with specific sequence numbers.""" + path = temp_file(b"x" * 1024) + + def _make(sequence: int, duration: float = 10.0) -> Chunk: + start = sequence * duration + return Chunk( + sequence=sequence, + start_time=start, + end_time=start + duration, + source_path=path, + duration=duration, + ) + + return _make + + +@pytest.fixture +def make_result(): + """Factory fixture for creating ChunkResults.""" + + def _make(sequence: int, success: bool = True, processing_time: float = 0.01) -> ChunkResult: + return ChunkResult( + sequence=sequence, + success=success, + processing_time=processing_time, + ) + + return _make diff --git a/tests/chunker/test_chunker.py b/tests/chunker/test_chunker.py new file mode 100644 index 0000000..7f132fa --- /dev/null +++ b/tests/chunker/test_chunker.py @@ -0,0 +1,149 @@ +""" +Tests for Chunker — time-based segmentation, chunk counts, sequence numbers, generator behavior. + +Demonstrates: TDD (Interview Topic 8) — parametrized tests, edge cases, mocking. +""" + +from unittest.mock import patch, MagicMock + +import pytest + +from core.chunker import Chunker +from core.chunker.exceptions import ChunkReadError + + +def mock_probe(duration): + """Create a mock probe_file that returns the given duration.""" + result = MagicMock() + result.duration = duration + return result + + +class TestChunker: + @patch("core.chunker.chunker.probe_file") + def test_basic_chunking(self, mock_pf, temp_file): + """File splits into expected number of time-based chunks.""" + path = temp_file(b"x" * 1000) + mock_pf.return_value = mock_probe(30.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + + assert len(chunks) == 3 + assert chunks[0].start_time == 0.0 + assert chunks[0].end_time == 10.0 + assert chunks[0].duration == 10.0 + assert chunks[1].start_time == 10.0 + assert chunks[2].start_time == 20.0 + + @patch("core.chunker.chunker.probe_file") + def test_sequence_numbers(self, mock_pf, temp_file): + """Chunks have sequential sequence numbers starting at 0.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(40.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + sequences = [c.sequence for c in chunks] + + assert sequences == [0, 1, 2, 3] + + @patch("core.chunker.chunker.probe_file") + def test_time_ranges(self, mock_pf, temp_file): + """Each chunk has correct start_time and end_time.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(25.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + + assert chunks[0].start_time == 0.0 + assert chunks[0].end_time == 10.0 + assert chunks[1].start_time == 10.0 + assert chunks[1].end_time == 20.0 + assert chunks[2].start_time == 20.0 + assert chunks[2].end_time == 25.0 # last chunk shorter + assert chunks[2].duration == 5.0 + + @patch("core.chunker.chunker.probe_file") + def test_expected_chunks_property(self, mock_pf, temp_file): + """expected_chunks calculates correctly before iteration.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(25.0) + + chunker = Chunker(path, chunk_duration=10.0) + assert chunker.expected_chunks == 3 # ceil(25/10) + + @patch("core.chunker.chunker.probe_file") + def test_source_path_on_chunks(self, mock_pf, temp_file): + """Each chunk carries the source file path.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(10.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + + assert all(c.source_path == path for c in chunks) + + def test_file_not_found(self): + """Non-existent file raises ChunkReadError.""" + with pytest.raises(ChunkReadError, match="File not found"): + Chunker("/nonexistent/file.mp4") + + @patch("core.chunker.chunker.probe_file") + def test_invalid_chunk_duration(self, mock_pf, temp_file): + """Zero or negative chunk_duration raises ValueError.""" + path = temp_file(b"x" * 100) + + with pytest.raises(ValueError, match="chunk_duration must be positive"): + Chunker(path, chunk_duration=0) + + with pytest.raises(ValueError, match="chunk_duration must be positive"): + Chunker(path, chunk_duration=-1) + + @patch("core.chunker.chunker.probe_file") + def test_generator_laziness(self, mock_pf, temp_file): + """Chunks are yielded lazily, not pre-loaded.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(30.0) + + chunker = Chunker(path, chunk_duration=10.0) + gen = chunker.chunks() + first = next(gen) + assert first.sequence == 0 + # Generator is not exhausted — remaining chunks still pending + + @pytest.mark.parametrize("duration,chunk_dur,expected", [ + (10.0, 10.0, 1), + (10.1, 10.0, 2), + (1.0, 1.0, 1), + (100.0, 1.0, 100), + (5.0, 100.0, 1), + ]) + @patch("core.chunker.chunker.probe_file") + def test_expected_chunks_parametrized(self, mock_pf, temp_file, duration, chunk_dur, expected): + """Parametrized: various duration/chunk_duration combos.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(duration) + chunker = Chunker(path, chunk_duration=chunk_dur) + assert chunker.expected_chunks == expected + + @patch("core.chunker.chunker.probe_file") + def test_exact_multiple(self, mock_pf, temp_file): + """Duration exactly divisible by chunk_duration.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(30.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + assert len(chunks) == 3 + assert all(c.duration == 10.0 for c in chunks) + + @patch("core.chunker.chunker.probe_file") + def test_probe_failure(self, mock_pf, temp_file): + """Probe failure raises ChunkReadError.""" + path = temp_file(b"x" * 100) + mock_pf.side_effect = Exception("ffprobe failed") + + with pytest.raises(ChunkReadError, match="Failed to probe"): + Chunker(path, chunk_duration=10.0) diff --git a/tests/chunker/test_collector.py b/tests/chunker/test_collector.py new file mode 100644 index 0000000..8dc1e64 --- /dev/null +++ b/tests/chunker/test_collector.py @@ -0,0 +1,103 @@ +""" +Tests for ResultCollector — ordered reassembly, out-of-order buffering, duplicates. + +Demonstrates: TDD (Interview Topic 8) — testing algorithms (heapq reassembly). +""" + +import pytest + +from core.chunker.collector import ResultCollector +from core.chunker.exceptions import ReassemblyError + + +class TestResultCollector: + def test_in_order_emission(self, make_result): + """Results arriving in order are emitted immediately.""" + collector = ResultCollector(total_chunks=3) + + emitted = collector.add(make_result(0)) + assert len(emitted) == 1 + assert emitted[0].sequence == 0 + + emitted = collector.add(make_result(1)) + assert len(emitted) == 1 + + emitted = collector.add(make_result(2)) + assert len(emitted) == 1 + + assert collector.is_complete + + def test_out_of_order_buffering(self, make_result): + """Out-of-order results are buffered until gaps fill.""" + collector = ResultCollector(total_chunks=3) + + # Arrive: 2, 0, 1 + emitted = collector.add(make_result(2)) + assert len(emitted) == 0 + assert collector.buffered_count == 1 + + emitted = collector.add(make_result(0)) + assert len(emitted) == 1 # Only 0 emitted, 1 still missing + + emitted = collector.add(make_result(1)) + assert len(emitted) == 2 # 1 and 2 now emittable + assert collector.is_complete + + def test_reverse_order(self, make_result): + """All results arrive in reverse — only last add emits everything.""" + collector = ResultCollector(total_chunks=4) + + for seq in [3, 2, 1]: + emitted = collector.add(make_result(seq)) + assert len(emitted) == 0 + + emitted = collector.add(make_result(0)) + assert len(emitted) == 4 + assert collector.is_complete + + def test_duplicate_raises(self, make_result): + """Duplicate sequence number raises ReassemblyError.""" + collector = ResultCollector(total_chunks=3) + collector.add(make_result(0)) + + with pytest.raises(ReassemblyError, match="Duplicate"): + collector.add(make_result(0)) + + def test_emitted_count(self, make_result): + """emitted_count tracks correctly.""" + collector = ResultCollector(total_chunks=3) + assert collector.emitted_count == 0 + + collector.add(make_result(0)) + assert collector.emitted_count == 1 + + collector.add(make_result(2)) # buffered + assert collector.emitted_count == 1 + + collector.add(make_result(1)) # releases 1 and 2 + assert collector.emitted_count == 3 + + def test_get_ordered_results(self, make_result): + """get_ordered_results returns all emitted results in order.""" + collector = ResultCollector(total_chunks=3) + collector.add(make_result(2)) + collector.add(make_result(0)) + collector.add(make_result(1)) + + ordered = collector.get_ordered_results() + assert [r.sequence for r in ordered] == [0, 1, 2] + + def test_avg_processing_time(self, make_result): + """Average processing time from sliding window.""" + collector = ResultCollector(total_chunks=2) + collector.add(make_result(0, processing_time=0.1)) + collector.add(make_result(1, processing_time=0.3)) + + assert abs(collector.avg_processing_time - 0.2) < 0.001 + + def test_not_complete_when_partial(self, make_result): + """is_complete is False until all chunks emitted.""" + collector = ResultCollector(total_chunks=3) + collector.add(make_result(0)) + collector.add(make_result(1)) + assert not collector.is_complete diff --git a/tests/chunker/test_exceptions.py b/tests/chunker/test_exceptions.py new file mode 100644 index 0000000..91ff59e --- /dev/null +++ b/tests/chunker/test_exceptions.py @@ -0,0 +1,69 @@ +""" +Tests for exception hierarchy — catch patterns, attributes. + +Demonstrates: TDD (Interview Topic 8) — testing exception design. +""" + +import pytest + +from core.chunker.exceptions import ( + ChunkChecksumError, + ChunkError, + ChunkReadError, + PipelineError, + ProcessingError, + ProcessorFailureError, + ProcessorTimeoutError, + ReassemblyError, +) + + +class TestExceptionHierarchy: + """Verify the exception class hierarchy and catch patterns.""" + + def test_pipeline_error_is_base(self): + """All chunker exceptions inherit from PipelineError.""" + assert issubclass(ChunkError, PipelineError) + assert issubclass(ProcessingError, PipelineError) + assert issubclass(ReassemblyError, PipelineError) + + def test_chunk_error_subtypes(self): + """ChunkReadError and ChunkChecksumError are ChunkErrors.""" + assert issubclass(ChunkReadError, ChunkError) + assert issubclass(ChunkChecksumError, ChunkError) + + def test_processing_error_subtypes(self): + """ProcessorTimeoutError and ProcessorFailureError are ProcessingErrors.""" + assert issubclass(ProcessorTimeoutError, ProcessingError) + assert issubclass(ProcessorFailureError, ProcessingError) + + def test_catch_pipeline_error_catches_all(self): + """Catching PipelineError catches any subtype.""" + with pytest.raises(PipelineError): + raise ChunkReadError("test") + + with pytest.raises(PipelineError): + raise ReassemblyError("test") + + def test_checksum_error_attributes(self): + """ChunkChecksumError carries sequence, expected, actual.""" + err = ChunkChecksumError(sequence=5, expected="aaa", actual="bbb") + assert err.sequence == 5 + assert err.expected == "aaa" + assert err.actual == "bbb" + assert "5" in str(err) + + def test_timeout_error_attributes(self): + """ProcessorTimeoutError carries sequence and timeout.""" + err = ProcessorTimeoutError(sequence=3, timeout=30.0) + assert err.sequence == 3 + assert err.timeout == 30.0 + + def test_failure_error_attributes(self): + """ProcessorFailureError carries sequence, retries, original error.""" + original = RuntimeError("boom") + err = ProcessorFailureError(sequence=1, retries=3, original_error=original) + assert err.sequence == 1 + assert err.retries == 3 + assert err.original_error is original + assert "boom" in str(err) diff --git a/tests/chunker/test_pipeline.py b/tests/chunker/test_pipeline.py new file mode 100644 index 0000000..e12e2d9 --- /dev/null +++ b/tests/chunker/test_pipeline.py @@ -0,0 +1,144 @@ +""" +Tests for Pipeline — end-to-end orchestration, stats, error handling. + +Demonstrates: TDD (Interview Topic 8) — integration testing with mocked FFmpeg probe. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from core.chunker import Pipeline +from core.chunker.exceptions import PipelineError + + +def mock_probe(duration): + """Create a mock ProbeResult with the given duration.""" + result = MagicMock() + result.duration = duration + return result + + +class TestPipeline: + @patch("core.chunker.chunker.probe_file") + def test_end_to_end(self, mock_pf, temp_file): + """Full pipeline processes a file successfully.""" + path = temp_file(b"x" * 4096) + mock_pf.return_value = mock_probe(40.0) + + result = Pipeline( + source=path, + chunk_duration=10.0, + num_workers=2, + processor_type="checksum", + ).run() + + assert result.total_chunks == 4 + assert result.processed == 4 + assert result.failed == 0 + assert result.elapsed_time > 0 + assert result.chunks_in_order is True + + @patch("core.chunker.chunker.probe_file") + def test_throughput_calculated(self, mock_pf, temp_file): + """Pipeline calculates throughput.""" + path = temp_file(b"x" * 10000) + mock_pf.return_value = mock_probe(30.0) + + result = Pipeline(source=path, chunk_duration=10.0, num_workers=2).run() + + assert result.throughput_mbps > 0 + + @patch("core.chunker.chunker.probe_file") + def test_worker_stats(self, mock_pf, temp_file): + """Pipeline reports per-worker stats.""" + path = temp_file(b"x" * 4000) + mock_pf.return_value = mock_probe(40.0) + + result = Pipeline( + source=path, chunk_duration=10.0, num_workers=2 + ).run() + + assert len(result.worker_stats) == 2 + for worker_id, stats in result.worker_stats.items(): + assert "processed" in stats + assert "errors" in stats + + def test_nonexistent_file(self): + """Non-existent file raises PipelineError.""" + with pytest.raises(PipelineError): + Pipeline(source="/nonexistent/file.mp4").run() + + @patch("core.chunker.chunker.probe_file") + def test_event_callback(self, mock_pf, temp_file): + """Pipeline emits events through callback.""" + path = temp_file(b"x" * 2048) + mock_pf.return_value = mock_probe(20.0) + events = [] + + def capture(event_type, data): + events.append(event_type) + + Pipeline( + source=path, + chunk_duration=10.0, + num_workers=1, + event_callback=capture, + ).run() + + assert "pipeline_start" in events + assert "pipeline_complete" in events + assert "chunk_queued" in events + + @patch("core.chunker.chunker.probe_file") + def test_simulated_decode_processor(self, mock_pf, temp_file): + """Pipeline works with simulated_decode processor.""" + path = temp_file(b"x" * 2048) + mock_pf.return_value = mock_probe(20.0) + + result = Pipeline( + source=path, + chunk_duration=10.0, + num_workers=2, + processor_type="simulated_decode", + ).run() + + assert result.total_chunks == 2 + assert result.failed == 0 + + @patch("core.chunker.chunker.probe_file") + def test_single_chunk_file(self, mock_pf, temp_file): + """Duration shorter than chunk_duration produces one chunk.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(5.0) + + result = Pipeline(source=path, chunk_duration=10.0).run() + + assert result.total_chunks == 1 + assert result.processed == 1 + + @patch("core.chunker.chunker.probe_file") + def test_retries_tracked(self, mock_pf, temp_file): + """Pipeline result tracks total retries.""" + path = temp_file(b"x" * 2048) + mock_pf.return_value = mock_probe(20.0) + + result = Pipeline(source=path, chunk_duration=10.0).run() + + assert result.retries >= 0 # Might be 0 if no failures + + @patch("core.chunker.chunker.probe_file") + def test_output_dir_and_chunk_files(self, mock_pf, temp_file): + """Pipeline tracks output_dir and chunk_files when set.""" + path = temp_file(b"x" * 1024) + mock_pf.return_value = mock_probe(10.0) + + result = Pipeline( + source=path, + chunk_duration=10.0, + processor_type="checksum", + ).run() + + # No output_dir set, so chunk_files should be empty + assert result.output_dir is None + assert result.chunk_files == [] diff --git a/tests/chunker/test_processor.py b/tests/chunker/test_processor.py new file mode 100644 index 0000000..68980f2 --- /dev/null +++ b/tests/chunker/test_processor.py @@ -0,0 +1,98 @@ +""" +Tests for Processor implementations — ChecksumProcessor, SimulatedDecodeProcessor, CompositeProcessor. + +Demonstrates: TDD (Interview Topic 8) — ABC contract, parametrized tests. +""" + +import pytest + +from core.chunker.exceptions import ChunkChecksumError +from core.chunker.models import Chunk +from core.chunker.processor import ( + ChecksumProcessor, + CompositeProcessor, + Processor, + SimulatedDecodeProcessor, +) + + +class TestChecksumProcessor: + def test_valid_time_range(self, sample_chunk): + """Valid time range passes.""" + proc = ChecksumProcessor() + result = proc.process(sample_chunk) + assert result.success is True + assert result.checksum_valid is True + assert result.processing_time > 0 + + def test_invalid_time_range(self): + """Invalid time range raises ChunkChecksumError.""" + chunk = Chunk( + sequence=0, + start_time=10.0, + end_time=10.0, # zero duration + source_path="/fake.mp4", + duration=0.0, + ) + proc = ChecksumProcessor() + with pytest.raises(ChunkChecksumError) as exc_info: + proc.process(chunk) + assert exc_info.value.sequence == 0 + + def test_sequence_preserved(self, make_chunk): + """Result carries the chunk's sequence number.""" + chunk = make_chunk(42) + proc = ChecksumProcessor() + result = proc.process(chunk) + assert result.sequence == 42 + + +class TestSimulatedDecodeProcessor: + def test_processes_successfully(self, sample_chunk): + """Simulated decode always succeeds.""" + proc = SimulatedDecodeProcessor(ms_per_second=1.0) + result = proc.process(sample_chunk) + assert result.success is True + assert result.processing_time > 0 + + def test_time_proportional_to_duration(self): + """Longer chunks take longer.""" + short = Chunk(0, 0.0, 1.0, "/fake.mp4", 1.0) + long = Chunk(1, 0.0, 10.0, "/fake.mp4", 10.0) + + proc = SimulatedDecodeProcessor(ms_per_second=50.0) + r_short = proc.process(short) + r_long = proc.process(long) + + assert r_long.processing_time > r_short.processing_time + + +class TestCompositeProcessor: + def test_chains_processors(self, sample_chunk): + """Composite runs all processors in sequence.""" + proc = CompositeProcessor([ + ChecksumProcessor(), + SimulatedDecodeProcessor(ms_per_second=1.0), + ]) + result = proc.process(sample_chunk) + assert result.success is True + + def test_stops_on_failure(self): + """If first processor raises, composite propagates the error.""" + bad_chunk = Chunk(0, 10.0, 10.0, "/fake.mp4", 0.0) # invalid range + proc = CompositeProcessor([ + ChecksumProcessor(), + SimulatedDecodeProcessor(ms_per_second=1.0), + ]) + with pytest.raises(ChunkChecksumError): + proc.process(bad_chunk) + + def test_requires_at_least_one(self): + """Empty processor list raises ValueError.""" + with pytest.raises(ValueError, match="at least one"): + CompositeProcessor([]) + + def test_is_processor(self): + """CompositeProcessor is a Processor.""" + proc = CompositeProcessor([ChecksumProcessor()]) + assert isinstance(proc, Processor) diff --git a/tests/chunker/test_queue.py b/tests/chunker/test_queue.py new file mode 100644 index 0000000..7ebee6b --- /dev/null +++ b/tests/chunker/test_queue.py @@ -0,0 +1,115 @@ +""" +Tests for ChunkQueue — backpressure, sentinel shutdown, timeout behavior. + +Demonstrates: TDD (Interview Topic 8) — concurrency testing. +""" + +import queue +import threading + +import pytest + +from core.chunker.queue import ChunkQueue + + +class TestChunkQueue: + def test_put_and_get(self, make_chunk): + """Basic put/get cycle.""" + q = ChunkQueue(maxsize=5) + chunk = make_chunk(0) + q.put(chunk) + result = q.get(timeout=1.0) + assert result.sequence == 0 + + def test_fifo_order(self, make_chunk): + """Items come out in FIFO order.""" + q = ChunkQueue(maxsize=5) + for i in range(3): + q.put(make_chunk(i)) + + for i in range(3): + assert q.get(timeout=1.0).sequence == i + + def test_close_returns_none(self, make_chunk): + """After close(), get() returns None (sentinel).""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + result = q.get(timeout=1.0) + assert result.sequence == 0 + + # Next get should hit sentinel + result = q.get(timeout=1.0) + assert result is None + + def test_close_propagates_to_multiple_consumers(self, make_chunk): + """Sentinel propagates: multiple consumers all get None.""" + q = ChunkQueue(maxsize=5) + q.close() + + # Multiple consumers should all see None + assert q.get(timeout=1.0) is None + assert q.get(timeout=1.0) is None + + def test_is_closed(self): + """is_closed reflects state.""" + q = ChunkQueue() + assert not q.is_closed + q.close() + assert q.is_closed + + def test_qsize(self, make_chunk): + """qsize tracks approximate queue depth.""" + q = ChunkQueue(maxsize=10) + assert q.qsize() == 0 + + q.put(make_chunk(0)) + q.put(make_chunk(1)) + assert q.qsize() == 2 + + q.get(timeout=1.0) + assert q.qsize() == 1 + + def test_backpressure_blocks(self, make_chunk): + """Put blocks when queue is full (backpressure).""" + q = ChunkQueue(maxsize=2) + q.put(make_chunk(0)) + q.put(make_chunk(1)) + + # Queue is full — put with short timeout should raise + with pytest.raises(queue.Full): + q.put(make_chunk(2), timeout=0.05) + + def test_get_timeout(self): + """Get on empty queue with timeout raises Empty.""" + q = ChunkQueue(maxsize=5) + + with pytest.raises(queue.Empty): + q.get(timeout=0.05) + + def test_concurrent_put_get(self, make_chunk): + """Producer/consumer threads work correctly.""" + q = ChunkQueue(maxsize=3) + results = [] + + def producer(): + for i in range(10): + q.put(make_chunk(i)) + q.close() + + def consumer(): + while True: + item = q.get(timeout=2.0) + if item is None: + break + results.append(item.sequence) + + t1 = threading.Thread(target=producer) + t2 = threading.Thread(target=consumer) + t1.start() + t2.start() + t1.join(timeout=5.0) + t2.join(timeout=5.0) + + assert sorted(results) == list(range(10)) diff --git a/tests/chunker/test_worker.py b/tests/chunker/test_worker.py new file mode 100644 index 0000000..12af394 --- /dev/null +++ b/tests/chunker/test_worker.py @@ -0,0 +1,127 @@ +""" +Tests for Worker — processing, retry with backoff, error handling. + +Demonstrates: TDD (Interview Topic 8) — mocking processors, testing retry logic. +""" + +from unittest.mock import MagicMock + +import pytest + +from core.chunker.models import Chunk, ChunkResult +from core.chunker.processor import Processor +from core.chunker.queue import ChunkQueue +from core.chunker.worker import Worker + + +class FailNTimesProcessor(Processor): + """Test processor that fails N times then succeeds.""" + + def __init__(self, fail_count: int): + self.fail_count = fail_count + self.call_count = 0 + + def process(self, chunk: Chunk) -> ChunkResult: + self.call_count += 1 + if self.call_count <= self.fail_count: + raise RuntimeError(f"Simulated failure #{self.call_count}") + return ChunkResult( + sequence=chunk.sequence, + success=True, + processing_time=0.001, + ) + + +class AlwaysFailProcessor(Processor): + """Test processor that always fails.""" + + def process(self, chunk: Chunk) -> ChunkResult: + raise RuntimeError("Always fails") + + +class TestWorker: + def test_processes_chunks(self, make_chunk): + """Worker processes all chunks from queue.""" + q = ChunkQueue(maxsize=5) + for i in range(3): + q.put(make_chunk(i)) + q.close() + + from core.chunker.processor import ChecksumProcessor + worker = Worker("w-0", q, ChecksumProcessor(), max_retries=0) + results = worker.run() + + assert len(results) == 3 + assert all(r.success for r in results) + + def test_retry_on_failure(self, make_chunk): + """Worker retries on processor failure.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + proc = FailNTimesProcessor(fail_count=2) + worker = Worker("w-0", q, proc, max_retries=3) + results = worker.run() + + assert len(results) == 1 + assert results[0].success is True + assert results[0].retries == 2 + assert proc.call_count == 3 # 2 failures + 1 success + + def test_max_retries_exceeded(self, make_chunk): + """Worker gives up after max retries.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + worker = Worker("w-0", q, AlwaysFailProcessor(), max_retries=2) + results = worker.run() + + assert len(results) == 1 + assert results[0].success is False + assert results[0].error is not None + assert worker.error_count == 1 + + def test_worker_id_on_results(self, make_chunk): + """Worker stamps its ID on results.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + from core.chunker.processor import ChecksumProcessor + worker = Worker("worker-7", q, ChecksumProcessor()) + results = worker.run() + + assert results[0].worker_id == "worker-7" + + def test_event_callback(self, make_chunk): + """Worker emits events via callback.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + events = [] + callback = MagicMock(side_effect=lambda t, d: events.append((t, d))) + + from core.chunker.processor import ChecksumProcessor + worker = Worker("w-0", q, ChecksumProcessor(), event_callback=callback) + worker.run() + + event_types = [e[0] for e in events] + assert "worker_status" in event_types + assert "chunk_processing" in event_types + assert "chunk_done" in event_types + + def test_processed_count(self, make_chunk): + """Worker tracks processed count.""" + q = ChunkQueue(maxsize=10) + for i in range(5): + q.put(make_chunk(i)) + q.close() + + from core.chunker.processor import ChecksumProcessor + worker = Worker("w-0", q, ChecksumProcessor()) + worker.run() + + assert worker.processed_count == 5 diff --git a/ui/chunker/index.html b/ui/chunker/index.html new file mode 100644 index 0000000..f333563 --- /dev/null +++ b/ui/chunker/index.html @@ -0,0 +1,12 @@ + + + + + + MPR Chunker Pipeline + + +
+ + + diff --git a/ui/chunker/package-lock.json b/ui/chunker/package-lock.json new file mode 100644 index 0000000..6633c19 --- /dev/null +++ b/ui/chunker/package-lock.json @@ -0,0 +1,1729 @@ +{ + "name": "mpr-chunker", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "mpr-chunker", + "version": "0.1.0", + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", + "@vitejs/plugin-react": "^4.2.0", + "typescript": "^5.3.0", + "vite": "^5.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", + "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", + "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.28.6", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", + "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", + "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.28.6", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", + "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", + "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.28.6", + "@babel/helper-validator-identifier": "^7.28.5", + "@babel/traverse": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", + "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", + "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.0.tgz", + "integrity": "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.27.1.tgz", + "integrity": "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.27.1.tgz", + "integrity": "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", + "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", + "integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz", + "integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz", + "integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz", + "integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz", + "integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz", + "integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz", + "integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz", + "integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz", + "integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz", + "integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz", + "integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz", + "integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz", + "integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz", + "integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz", + "integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz", + "integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz", + "integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz", + "integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz", + "integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz", + "integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz", + "integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz", + "integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz", + "integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz", + "integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz", + "integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.28", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz", + "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.7", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.7.tgz", + "integrity": "sha512-1ghYO3HnxGec0TCGBXiDLVns4eCSx4zJpxnHrlqFQajmhfKMQBzUGDdkMK7fUW7PTHTeLf+j87aTuKuuwWzMGw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001778", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001778.tgz", + "integrity": "sha512-PN7uxFL+ExFJO61aVmP1aIEG4i9whQd4eoSCebav62UwDyp5OHh06zN4jqKSMePVgxHifCw1QJxdRkA1Pisekg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.313", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.313.tgz", + "integrity": "sha512-QBMrTWEf00GXZmJyx2lbYD45jpI3TUFnNIzJ5BBc8piGUDwMPa1GV6HJWTZVvY/eiN3fSopl7NRbgGp9sZ9LTA==", + "dev": true, + "license": "ISC" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.36", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", + "integrity": "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.8", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", + "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", + "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.59.0", + "@rollup/rollup-android-arm64": "4.59.0", + "@rollup/rollup-darwin-arm64": "4.59.0", + "@rollup/rollup-darwin-x64": "4.59.0", + "@rollup/rollup-freebsd-arm64": "4.59.0", + "@rollup/rollup-freebsd-x64": "4.59.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", + "@rollup/rollup-linux-arm-musleabihf": "4.59.0", + "@rollup/rollup-linux-arm64-gnu": "4.59.0", + "@rollup/rollup-linux-arm64-musl": "4.59.0", + "@rollup/rollup-linux-loong64-gnu": "4.59.0", + "@rollup/rollup-linux-loong64-musl": "4.59.0", + "@rollup/rollup-linux-ppc64-gnu": "4.59.0", + "@rollup/rollup-linux-ppc64-musl": "4.59.0", + "@rollup/rollup-linux-riscv64-gnu": "4.59.0", + "@rollup/rollup-linux-riscv64-musl": "4.59.0", + "@rollup/rollup-linux-s390x-gnu": "4.59.0", + "@rollup/rollup-linux-x64-gnu": "4.59.0", + "@rollup/rollup-linux-x64-musl": "4.59.0", + "@rollup/rollup-openbsd-x64": "4.59.0", + "@rollup/rollup-openharmony-arm64": "4.59.0", + "@rollup/rollup-win32-arm64-msvc": "4.59.0", + "@rollup/rollup-win32-ia32-msvc": "4.59.0", + "@rollup/rollup-win32-x64-gnu": "4.59.0", + "@rollup/rollup-win32-x64-msvc": "4.59.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/ui/chunker/package.json b/ui/chunker/package.json new file mode 100644 index 0000000..548d863 --- /dev/null +++ b/ui/chunker/package.json @@ -0,0 +1,22 @@ +{ + "name": "mpr-chunker", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", + "@vitejs/plugin-react": "^4.2.0", + "typescript": "^5.3.0", + "vite": "^5.0.0" + } +} diff --git a/ui/chunker/src/App.css b/ui/chunker/src/App.css new file mode 100644 index 0000000..c4a0381 --- /dev/null +++ b/ui/chunker/src/App.css @@ -0,0 +1,735 @@ +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, + "Fira Code", monospace, sans-serif; + background: #0f0f0f; + color: #e0e0e0; + font-size: 14px; +} + +/* ---- Layout ---- */ + +.app { + display: flex; + flex-direction: column; + height: 100vh; +} + +.header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1.25rem; + background: #1a1a1a; + border-bottom: 1px solid #2a2a2a; +} + +.header h1 { + font-size: 1.1rem; + font-weight: 600; + letter-spacing: -0.01em; +} + +.connection-status { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; + color: #666; +} + +.dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: #555; +} + +.dot.connected { + background: #10b981; + box-shadow: 0 0 6px #10b981; +} + +.error-banner { + padding: 0.5rem 1.25rem; + background: #7f1d1d; + color: #fca5a5; + font-size: 0.85rem; +} + +.layout { + display: flex; + flex: 1; + overflow: hidden; +} + +.sidebar { + width: 300px; + background: #141414; + border-right: 1px solid #2a2a2a; + overflow-y: auto; +} + +.main { + flex: 1; + overflow-y: auto; + padding: 1rem; + display: flex; + flex-direction: column; + gap: 1rem; +} + +.main-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; +} + +.main-left, +.main-right { + display: flex; + flex-direction: column; + gap: 1rem; +} + +/* ---- Panel shared ---- */ + +.panel-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.75rem; +} + +.panel-header h2 { + font-size: 0.85rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: #888; +} + +.badge-row { + display: flex; + gap: 0.25rem; +} + +/* ---- Topic Badge ---- */ + +.topic-badge { + display: inline-flex; + align-items: center; + gap: 0.25rem; + padding: 0.15rem 0.5rem; + font-size: 0.65rem; + background: #1e293b; + border: 1px solid #334155; + border-radius: 12px; + cursor: pointer; + transition: all 0.2s; + flex-shrink: 0; +} + +.topic-badge:hover { + border-color: #3b82f6; +} + +.topic-badge.expanded { + flex-direction: column; + align-items: flex-start; + border-radius: 8px; + padding: 0.5rem; + position: relative; + z-index: 10; + background: #1e293b; +} + +.topic-number { + color: #3b82f6; + font-weight: 700; +} + +.topic-title { + color: #94a3b8; +} + +.topic-detail { + margin-top: 0.25rem; + font-size: 0.7rem; + line-height: 1.4; +} + +.topic-detail p { + color: #cbd5e1; + margin-bottom: 0.25rem; +} + +.topic-detail code { + color: #10b981; + font-size: 0.65rem; +} + +/* ---- Asset List ---- */ + +.scan-button { + padding: 0.25rem 0.5rem; + font-size: 0.7rem; + background: #1e293b; + color: #94a3b8; + border: 1px solid #334155; + border-radius: 4px; + cursor: pointer; + transition: all 0.2s; +} + +.scan-button:hover:not(:disabled) { + background: #334155; + color: #e0e0e0; +} + +.scan-button:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.asset-list { + list-style: none; + max-height: 200px; + overflow-y: auto; + margin-bottom: 0.75rem; +} + +.asset-item { + padding: 0.4rem 0.5rem; + cursor: pointer; + border-left: 2px solid transparent; + transition: all 0.15s; + display: flex; + flex-direction: column; + gap: 0.1rem; +} + +.asset-item:hover { + background: #1a1a1a; +} + +.asset-item.selected { + background: #1e293b; + border-left-color: #3b82f6; +} + +.asset-filename { + font-size: 0.8rem; + color: #e0e0e0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.asset-meta { + font-size: 0.65rem; + color: #555; +} + +.asset-empty { + font-size: 0.8rem; + color: #444; + padding: 0.75rem 0.5rem; + text-align: center; +} + +.selected-asset-info { + padding: 0.5rem; + background: #1e293b; + border: 1px solid #334155; + border-radius: 4px; + margin-bottom: 0.75rem; +} + +.asset-detail { + display: block; + font-size: 0.8rem; + color: #e0e0e0; + font-weight: 500; +} + +.asset-detail-meta { + display: block; + font-size: 0.65rem; + color: #64748b; + margin-top: 0.15rem; +} + +/* ---- Config Panel ---- */ + +.config-panel { + padding: 1rem; +} + +.config-field { + margin-bottom: 0.75rem; +} + +.config-field label { + display: block; + font-size: 0.75rem; + color: #888; + margin-bottom: 0.25rem; +} + +.config-field .default { + color: #555; + font-style: italic; +} + +.config-field input, +.config-field select { + width: 100%; + padding: 0.4rem 0.5rem; + font-size: 0.8rem; + background: #222; + color: #e0e0e0; + border: 1px solid #333; + border-radius: 4px; +} + +.config-field input:focus, +.config-field select:focus { + outline: none; + border-color: #3b82f6; +} + +.start-button { + width: 100%; + padding: 0.5rem; + font-size: 0.85rem; + background: #10b981; + color: #000; + border: none; + border-radius: 4px; + cursor: pointer; + font-weight: 600; + margin-top: 0.5rem; + transition: background 0.2s; +} + +.start-button:hover:not(:disabled) { + background: #059669; +} + +.start-button:disabled { + background: #333; + color: #666; + cursor: not-allowed; +} + +/* ---- Pipeline Diagram ---- */ + +.pipeline-diagram { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.stage-flow { + display: flex; + align-items: center; + gap: 0; + overflow-x: auto; +} + +.stage-wrapper { + display: flex; + align-items: center; +} + +.stage { + padding: 0.5rem 0.75rem; + background: #1a1a1a; + border: 1px solid #333; + border-radius: 6px; + text-align: center; + min-width: 120px; + transition: all 0.3s; +} + +.stage.active { + border-color: #3b82f6; + background: #1e293b; + box-shadow: 0 0 12px rgba(59, 130, 246, 0.2); +} + +.stage-label { + font-size: 0.8rem; + font-weight: 600; + color: #e0e0e0; +} + +.stage-sub { + font-size: 0.65rem; + color: #666; + margin-top: 0.15rem; +} + +.stage-arrow { + width: 24px; + height: 2px; + background: #444; + position: relative; +} + +.stage-arrow::after { + content: ""; + position: absolute; + right: 0; + top: -3px; + border: 4px solid transparent; + border-left: 6px solid #444; +} + +.processor-hierarchy { + margin-top: 0.75rem; + padding-top: 0.75rem; + border-top: 1px solid #222; +} + +.hierarchy-title { + font-size: 0.7rem; + color: #666; + margin-bottom: 0.35rem; + font-style: italic; +} + +.hierarchy-children { + display: flex; + gap: 0.5rem; + flex-wrap: wrap; +} + +.hierarchy-node { + font-size: 0.7rem; + padding: 0.15rem 0.5rem; + background: #1a1a1a; + border: 1px solid #333; + border-radius: 4px; + color: #94a3b8; +} + +/* ---- Chunk Grid ---- */ + +.chunk-grid-panel { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.chunk-count { + font-size: 0.7rem; + color: #555; + font-weight: 400; +} + +.chunk-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(32px, 1fr)); + gap: 3px; + max-height: 200px; + overflow-y: auto; +} + +.chunk-cell { + aspect-ratio: 1; + display: flex; + align-items: center; + justify-content: center; + font-size: 0.55rem; + color: rgba(255, 255, 255, 0.6); + border-radius: 3px; + transition: background 0.3s; +} + +.chunk-legend { + display: flex; + gap: 0.75rem; + margin-top: 0.5rem; + flex-wrap: wrap; +} + +.legend-item { + display: flex; + align-items: center; + gap: 0.25rem; + font-size: 0.65rem; + color: #888; +} + +.legend-dot { + width: 8px; + height: 8px; + border-radius: 2px; +} + +/* ---- Worker Panel ---- */ + +.worker-panel { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.worker-cards { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.worker-card { + padding: 0.5rem 0.75rem; + background: #1a1a1a; + border: 1px solid #2a2a2a; + border-radius: 6px; +} + +.worker-header { + display: flex; + justify-content: space-between; + align-items: center; +} + +.worker-name { + font-size: 0.8rem; + font-weight: 500; +} + +.worker-state { + font-size: 0.7rem; + text-transform: uppercase; + font-weight: 600; +} + +.worker-chunk { + font-size: 0.7rem; + color: #555; + margin-top: 0.15rem; +} + +.worker-stats { + display: flex; + gap: 0.75rem; + font-size: 0.65rem; + color: #555; + margin-top: 0.25rem; +} + +.worker-empty { + font-size: 0.8rem; + color: #444; + text-align: center; + padding: 1rem; +} + +/* ---- Queue Gauge ---- */ + +.queue-gauge { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.gauge-row { + margin-bottom: 0.5rem; +} + +.gauge-label { + font-size: 0.75rem; + color: #888; + margin-bottom: 0.25rem; +} + +.gauge-value { + color: #e0e0e0; + font-weight: 600; +} + +.gauge-bar { + height: 8px; + background: #222; + border-radius: 4px; + overflow: hidden; +} + +.gauge-fill { + height: 100%; + border-radius: 4px; + transition: width 0.3s, background 0.3s; +} + +.gauge-note { + font-size: 0.65rem; + color: #555; +} + +/* ---- Stats Panel ---- */ + +.stats-panel { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.stats-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 0.5rem; +} + +.stat { + text-align: center; + padding: 0.5rem; + background: #1a1a1a; + border-radius: 6px; +} + +.stat-value { + font-size: 1.1rem; + font-weight: 700; + color: #e0e0e0; +} + +.stat-label { + font-size: 0.6rem; + color: #666; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-top: 0.15rem; +} + +.test-info { + margin-top: 0.75rem; + padding-top: 0.5rem; + border-top: 1px solid #222; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.test-badge { + font-size: 0.65rem; + padding: 0.15rem 0.4rem; + background: #10b981; + color: #000; + border-radius: 3px; + font-weight: 600; +} + +.test-note { + font-size: 0.65rem; + color: #555; +} + +/* ---- Error Log ---- */ + +.error-log { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.error-count { + font-size: 0.7rem; + background: #7f1d1d; + color: #fca5a5; + padding: 0.1rem 0.4rem; + border-radius: 8px; + font-weight: 400; +} + +.exception-tree { + margin-bottom: 0.75rem; + padding: 0.5rem; + background: #1a1a1a; + border-radius: 6px; + font-size: 0.7rem; + font-family: "Fira Code", monospace; +} + +.tree-node { + color: #94a3b8; + padding: 0.1rem 0; +} + +.tree-node.root { + color: #f59e0b; + font-weight: 600; +} + +.tree-node.leaf { + color: #64748b; +} + +.tree-children { + padding-left: 1rem; + border-left: 1px solid #333; + margin-left: 0.5rem; +} + +.tree-grandchildren { + padding-left: 1rem; + border-left: 1px solid #333; + margin-left: 0.5rem; +} + +.error-entries { + max-height: 150px; + overflow-y: auto; +} + +.error-empty { + font-size: 0.8rem; + color: #444; + text-align: center; + padding: 0.5rem; +} + +.error-entry { + display: flex; + gap: 0.5rem; + align-items: center; + padding: 0.35rem 0; + border-bottom: 1px solid #1a1a1a; + font-size: 0.7rem; + flex-wrap: wrap; +} + +.error-type { + color: #ef4444; + font-weight: 500; +} + +.error-seq { + color: #f59e0b; +} + +.error-worker { + color: #3b82f6; +} + +.error-msg { + color: #888; + flex: 1; +} + +.error-retries { + color: #f97316; + font-size: 0.65rem; +} diff --git a/ui/chunker/src/App.tsx b/ui/chunker/src/App.tsx new file mode 100644 index 0000000..b81a895 --- /dev/null +++ b/ui/chunker/src/App.tsx @@ -0,0 +1,245 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; +import "./App.css"; +import { createChunkJob, getAssets, scanMediaFolder } from "./api"; +import { ChunkGrid } from "./components/ChunkGrid"; +import { ConfigPanel } from "./components/ConfigPanel"; +import { ErrorLog } from "./components/ErrorLog"; +import { PipelineDiagram } from "./components/PipelineDiagram"; +import { QueueGauge } from "./components/QueueGauge"; +import { StatsPanel } from "./components/StatsPanel"; +import { WorkerPanel } from "./components/WorkerPanel"; +import { useEventStream } from "./hooks/useEventStream"; +import type { + ChunkInfo, + ErrorEntry, + MediaAsset, + PipelineConfig, + PipelineStats, + WorkerInfo, +} from "./types"; + +export default function App() { + const [jobId, setJobId] = useState(null); + const [running, setRunning] = useState(false); + const [error, setError] = useState(null); + + // Asset state + const [assets, setAssets] = useState([]); + const [selectedAsset, setSelectedAsset] = useState(null); + const [scanning, setScanning] = useState(false); + + const { events, connected, done } = useEventStream(jobId); + + // Load assets on mount + useEffect(() => { + getAssets() + .then((data) => setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename)))) + .catch((e) => setError(e instanceof Error ? e.message : "Failed to load assets")); + }, []); + + const handleScan = useCallback(async () => { + setScanning(true); + setError(null); + try { + await scanMediaFolder(); + const data = await getAssets(); + setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename))); + } catch (e) { + setError(e instanceof Error ? e.message : "Scan failed"); + } finally { + setScanning(false); + } + }, []); + + // Derive state from events + const { chunks, workers, stats, errors, activeStage, queueSize } = + useMemo(() => { + const chunkMap = new Map(); + const workerMap = new Map(); + const errorList: ErrorEntry[] = []; + let totalChunks = 0; + let processed = 0; + let failed = 0; + let retries = 0; + let elapsed = 0; + let throughput = 0; + let queueSize = 0; + let stage = "pending"; + + for (const evt of events) { + if (evt.total_chunks) totalChunks = evt.total_chunks; + if (evt.processed_chunks) processed = evt.processed_chunks; + if (evt.failed_chunks) failed = evt.failed_chunks; + if (evt.elapsed) elapsed = evt.elapsed; + if (evt.throughput_mbps) throughput = evt.throughput_mbps; + if (evt.queue_size !== undefined) queueSize = evt.queue_size; + if (evt.status && evt.status !== "waiting") stage = evt.status; + + // Track chunks + if (evt.sequence !== undefined) { + const existing = chunkMap.get(evt.sequence) || { + sequence: evt.sequence, + state: "pending" as const, + }; + + if (evt.status === "chunking" || evt.status === "pending") { + existing.state = "queued"; + } else if (evt.status === "processing") { + existing.state = "processing"; + if (evt.worker_id) existing.worker_id = evt.worker_id; + } else if (evt.status === "completed") { + existing.state = "done"; + if (evt.processing_time) + existing.processing_time = evt.processing_time; + if (evt.retries) existing.retries = evt.retries; + } else if (evt.status === "failed") { + existing.state = "error"; + if (evt.error) existing.error = evt.error; + } + + if (evt.size) existing.size = evt.size; + chunkMap.set(evt.sequence, existing); + } + + // Track workers + if (evt.worker_id) { + const w = workerMap.get(evt.worker_id) || { + worker_id: evt.worker_id, + state: "idle" as const, + processed: 0, + errors: 0, + retries: 0, + }; + + if (evt.state === "processing") { + w.state = "processing"; + w.current_chunk = evt.sequence; + } else if (evt.state === "idle") { + w.state = "idle"; + w.current_chunk = undefined; + } else if (evt.state === "stopped") { + w.state = "stopped"; + } + + if (evt.success !== undefined) { + if (evt.success) w.processed++; + else w.errors++; + } + if (evt.retries) { + retries += evt.retries; + w.retries += evt.retries; + } + + workerMap.set(evt.worker_id, w); + } + + // Track errors + if (evt.error) { + errorList.push({ + timestamp: Date.now(), + sequence: evt.sequence, + worker_id: evt.worker_id, + error: evt.error, + retries: evt.retries, + event_type: evt.status || "error", + }); + } + } + + const statsObj: PipelineStats = { + total_chunks: totalChunks, + processed, + failed, + retries, + elapsed, + throughput_mbps: throughput, + queue_size: queueSize, + }; + + return { + chunks: Array.from(chunkMap.values()).sort( + (a, b) => a.sequence - b.sequence + ), + workers: Array.from(workerMap.values()), + stats: statsObj, + errors: errorList, + activeStage: stage, + queueSize, + }; + }, [events]); + + const handleStart = useCallback(async (config: PipelineConfig) => { + setError(null); + setRunning(true); + try { + const result = await createChunkJob(config); + setJobId(result.id); + } catch (e) { + setError(e instanceof Error ? e.message : "Failed to start"); + setRunning(false); + } + }, []); + + // Reset running state when done + if (done && running) { + setRunning(false); + } + + return ( +
+
+

MPR Chunker Pipeline

+
+ {jobId && ( + + )} + + {!jobId + ? "Configure and launch" + : connected + ? "Streaming" + : done + ? "Complete" + : "Connecting..."} + +
+
+ + {error &&
{error}
} + +
+ + +
+ + +
+
+ + +
+
+ + + +
+
+
+
+
+ ); +} diff --git a/ui/chunker/src/api.ts b/ui/chunker/src/api.ts new file mode 100644 index 0000000..c3d4f99 --- /dev/null +++ b/ui/chunker/src/api.ts @@ -0,0 +1,72 @@ +/** + * GraphQL API client for the chunker UI. + */ + +import type { MediaAsset } from "./types"; + +const GRAPHQL_URL = "/api/graphql"; + +async function gql(query: string, variables?: Record): Promise { + const response = await fetch(GRAPHQL_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query, variables }), + }); + + const json = await response.json(); + + if (json.errors?.length) { + throw new Error(json.errors[0].message); + } + + return json.data as T; +} + +/** Fetch all media assets. */ +export async function getAssets(): Promise { + const data = await gql<{ assets: MediaAsset[] }>(` + query { + assets { + id filename file_path status error_message file_size duration + video_codec audio_codec width height framerate bitrate + properties comments tags created_at updated_at + } + } + `); + return data.assets; +} + +/** Scan media/in/ folder for new files. */ +export async function scanMediaFolder(): Promise<{ + found: number; + registered: number; + skipped: number; + files: string[]; +}> { + const data = await gql<{ scan_media_folder: { found: number; registered: number; skipped: number; files: string[] } }>(` + mutation { + scan_media_folder { found registered skipped files } + } + `); + return data.scan_media_folder; +} + +/** Create a chunk job via GraphQL mutation. */ +export async function createChunkJob(config: { + source_asset_id: string; + chunk_duration: number; + num_workers: number; + max_retries: number; + processor_type: string; +}): Promise<{ id: string }> { + const data = await gql<{ create_chunk_job: { id: string; status: string } }>(` + mutation CreateChunkJob($input: CreateChunkJobInput!) { + create_chunk_job(input: $input) { + id + status + } + } + `, { input: config }); + + return data.create_chunk_job; +} diff --git a/ui/chunker/src/components/ChunkGrid.tsx b/ui/chunker/src/components/ChunkGrid.tsx new file mode 100644 index 0000000..1f63db7 --- /dev/null +++ b/ui/chunker/src/components/ChunkGrid.tsx @@ -0,0 +1,59 @@ +import type { ChunkInfo } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + chunks: ChunkInfo[]; + totalChunks: number; +} + +const STATE_COLORS: Record = { + pending: "#333", + queued: "#f59e0b", + processing: "#3b82f6", + done: "#10b981", + error: "#ef4444", + retry: "#f97316", +}; + +/** + * Grid of chunks colored by processing state. + * Chunks appear incrementally as the generator yields them. + * Interview Topic 3: Generators & iteration. + */ +export function ChunkGrid({ chunks, totalChunks }: Props) { + return ( +
+
+

+ Chunks{" "} + + {chunks.length} / {totalChunks || "?"} + +

+ +
+
+ {chunks.map((chunk) => ( +
+ {chunk.sequence} +
+ ))} +
+
+ {Object.entries(STATE_COLORS).map(([state, color]) => ( + + + {state} + + ))} +
+
+ ); +} diff --git a/ui/chunker/src/components/ConfigPanel.tsx b/ui/chunker/src/components/ConfigPanel.tsx new file mode 100644 index 0000000..f3792b5 --- /dev/null +++ b/ui/chunker/src/components/ConfigPanel.tsx @@ -0,0 +1,172 @@ +import { useState } from "react"; +import type { MediaAsset, PipelineConfig } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + onStart: (config: PipelineConfig) => void; + running: boolean; + assets: MediaAsset[]; + selectedAsset: MediaAsset | null; + onSelectAsset: (asset: MediaAsset) => void; + onScan: () => void; + scanning: boolean; +} + +/** + * Pipeline configuration form with file browser. + * Each parameter shows its default — Interview Topic 1: Function params & defaults. + */ +export function ConfigPanel({ + onStart, + running, + assets, + selectedAsset, + onSelectAsset, + onScan, + scanning, +}: Props) { + const [chunkDuration, setChunkDuration] = useState(10.0); + const [numWorkers, setNumWorkers] = useState(4); + const [maxRetries, setMaxRetries] = useState(3); + const [processorType, setProcessorType] = useState< + "ffmpeg" | "checksum" | "simulated_decode" | "composite" + >("ffmpeg"); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (!selectedAsset) return; + onStart({ + source_asset_id: selectedAsset.id, + chunk_duration: chunkDuration, + num_workers: numWorkers, + max_retries: maxRetries, + processor_type: processorType, + }); + }; + + const formatSize = (bytes: number | null) => { + if (!bytes) return "—"; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + }; + + const formatDuration = (seconds: number | null) => { + if (!seconds) return "—"; + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m}:${s.toString().padStart(2, "0")}`; + }; + + return ( +
+ {/* Asset Browser */} +
+

Assets

+ +
+ +
    + {assets.length === 0 ? ( +
  • No assets — click Scan Folder
    • + ) : ( + assets.map((asset) => ( +
  • onSelectAsset(asset)} + title={asset.filename} + > + {asset.filename} + + {formatSize(asset.file_size)} · {formatDuration(asset.duration)} + +
    • + )) + )} +
+ + {selectedAsset && ( +
+ {selectedAsset.filename} + + {selectedAsset.video_codec} · {selectedAsset.width}x{selectedAsset.height} · {formatDuration(selectedAsset.duration)} + +
+ )} + + {/* Pipeline Config */} +
+

Pipeline Config

+ +
+
+
+ + +
+
+ + setNumWorkers(Number(e.target.value))} + /> +
+
+ + setMaxRetries(Number(e.target.value))} + /> +
+
+ + +
+ +
+
+ ); +} diff --git a/ui/chunker/src/components/ErrorLog.tsx b/ui/chunker/src/components/ErrorLog.tsx new file mode 100644 index 0000000..421705f --- /dev/null +++ b/ui/chunker/src/components/ErrorLog.tsx @@ -0,0 +1,63 @@ +import type { ErrorEntry } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + errors: ErrorEntry[]; +} + +/** + * Error and retry event log. + * Shows exception types, retry counts, backoff delays. + * Interview Topic 7: Exception handling & resilient code. + */ +export function ErrorLog({ errors }: Props) { + return ( +
+
+

+ Errors & Retries{" "} + {errors.length} +

+ +
+
+
PipelineError
+
+
ChunkError
+
+
ChunkReadError
+
ChunkChecksumError
+
+
ProcessingError
+
+
ProcessorTimeoutError
+
ProcessorFailureError
+
+
ReassemblyError
+
+
+
+ {errors.length === 0 && ( +
No errors recorded
+ )} + {errors.map((entry, i) => ( +
+ {entry.event_type} + {entry.sequence !== undefined && ( + chunk #{entry.sequence} + )} + {entry.worker_id && ( + {entry.worker_id} + )} + {entry.error} + {entry.retries !== undefined && entry.retries > 0 && ( + + {entry.retries} retries + + )} +
+ ))} +
+
+ ); +} diff --git a/ui/chunker/src/components/PipelineDiagram.tsx b/ui/chunker/src/components/PipelineDiagram.tsx new file mode 100644 index 0000000..2f8ef8e --- /dev/null +++ b/ui/chunker/src/components/PipelineDiagram.tsx @@ -0,0 +1,50 @@ +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + activeStage: string; +} + +const STAGES = [ + { id: "chunking", label: "Chunker", sub: "File -> Chunks (generator)" }, + { id: "queued", label: "ChunkQueue", sub: "Bounded queue (backpressure)" }, + { id: "processing", label: "WorkerPool", sub: "ThreadPoolExecutor" }, + { id: "collecting", label: "ResultCollector", sub: "heapq reassembly" }, + { id: "completed", label: "PipelineResult", sub: "Aggregate stats" }, +]; + +/** + * Visual flow diagram of pipeline stages. + * Highlights the currently active stage. + * Interview Topic 4: OOP design — shows class hierarchy. + */ +export function PipelineDiagram({ activeStage }: Props) { + return ( +
+
+

Pipeline Flow

+ +
+
+ {STAGES.map((stage, i) => ( +
+
+
{stage.label}
+
{stage.sub}
+
+ {i < STAGES.length - 1 &&
} +
+ ))} +
+
+
Processor ABC
+
+ ChecksumProcessor + SimulatedDecodeProcessor + CompositeProcessor +
+
+
+ ); +} diff --git a/ui/chunker/src/components/QueueGauge.tsx b/ui/chunker/src/components/QueueGauge.tsx new file mode 100644 index 0000000..305a597 --- /dev/null +++ b/ui/chunker/src/components/QueueGauge.tsx @@ -0,0 +1,46 @@ +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + current: number; + max: number; + buffered: number; +} + +/** + * Queue fill level gauge + collector heap buffer. + * Interview Topic 5: Data structures — queue.Queue, heapq, deque. + */ +export function QueueGauge({ current, max, buffered }: Props) { + const fillPct = max > 0 ? Math.min((current / max) * 100, 100) : 0; + + return ( +
+
+

Queue & Buffer

+ +
+
+
+ Queue {current}/{max} +
+
+
80 ? "#ef4444" : "#3b82f6", + }} + /> +
+
+
+
+ Heap Buffer {buffered} +
+
+ Out-of-order results waiting for gaps to fill +
+
+
+ ); +} diff --git a/ui/chunker/src/components/StatsPanel.tsx b/ui/chunker/src/components/StatsPanel.tsx new file mode 100644 index 0000000..8dd2f24 --- /dev/null +++ b/ui/chunker/src/components/StatsPanel.tsx @@ -0,0 +1,59 @@ +import type { PipelineStats } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + stats: PipelineStats; +} + +/** + * Throughput, timing, and error stats. + * Interview Topic 6: Algorithms — throughput calculation over sliding window. + * Interview Topic 8: TDD — test count and coverage. + */ +export function StatsPanel({ stats }: Props) { + return ( +
+
+

Stats

+
+ + +
+
+
+
+
{stats.total_chunks}
+
Total Chunks
+
+
+
{stats.processed}
+
Processed
+
+
+
{stats.failed}
+
Failed
+
+
+
{stats.retries}
+
Retries
+
+
+
+ {stats.throughput_mbps.toFixed(2)} +
+
MB/s
+
+
+
{stats.elapsed.toFixed(2)}s
+
Elapsed
+
+
+
+ 64 tests + + 7 test files · pytest · parametrized + +
+
+ ); +} diff --git a/ui/chunker/src/components/TopicBadge.tsx b/ui/chunker/src/components/TopicBadge.tsx new file mode 100644 index 0000000..c21f9e9 --- /dev/null +++ b/ui/chunker/src/components/TopicBadge.tsx @@ -0,0 +1,86 @@ +import { useState } from "react"; +import type { InterviewTopic } from "../types"; + +/** + * Expandable pill badge annotating an interview topic. + * Click to expand and see description + code reference. + */ +export function TopicBadge({ topic }: { topic: InterviewTopic }) { + const [expanded, setExpanded] = useState(false); + + return ( +
setExpanded(!expanded)} + > + #{topic.number} + {topic.title} + {expanded && ( +
+

{topic.description}

+ {topic.code_ref} +
+ )} +
+ ); +} + +/** Pre-defined topics mapped to pipeline components. */ +export const TOPICS: Record = { + params: { + number: 1, + title: "Function Params & Defaults", + description: + "Each pipeline parameter has a sensible default (chunk_duration=10s, num_workers=4, max_retries=3). Tweaking them changes pipeline behavior.", + code_ref: "core/chunker/pipeline.py — Pipeline.__init__()", + }, + concurrency: { + number: 2, + title: "Concurrency (Threading)", + description: + "Workers run in a ThreadPoolExecutor. The queue coordinates work between producer and consumer threads.", + code_ref: "core/chunker/pool.py — WorkerPool, ThreadPoolExecutor", + }, + iteration: { + number: 3, + title: "Generators & Iteration", + description: + "Chunks are yielded lazily via a generator — the file is never fully loaded into memory.", + code_ref: "core/chunker/chunker.py — Chunker.chunks() generator", + }, + oop: { + number: 4, + title: "OOP Design (ABC)", + description: + "Processor is an abstract base class. ChecksumProcessor, SimulatedDecodeProcessor, and CompositeProcessor inherit from it.", + code_ref: "core/chunker/processor.py — Processor ABC hierarchy", + }, + datastructures: { + number: 5, + title: "Data Structures", + description: + "Bounded queue.Queue for backpressure, heapq min-heap for ordered reassembly, deque for sliding-window throughput.", + code_ref: "core/chunker/queue.py, collector.py, models.py", + }, + algorithms: { + number: 6, + title: "Algorithms & Sorting", + description: + "ResultCollector uses a min-heap to reassemble chunks in sequence order, even when they arrive out of order.", + code_ref: "core/chunker/collector.py — heapq-based reassembly", + }, + exceptions: { + number: 7, + title: "Exception Handling", + description: + "PipelineError hierarchy with typed exceptions. Workers retry with exponential backoff before giving up.", + code_ref: "core/chunker/exceptions.py, worker.py — retry logic", + }, + testing: { + number: 8, + title: "TDD & Unit Testing", + description: + "64 tests covering every module. Parametrized tests, fixtures, edge cases, concurrency tests.", + code_ref: "tests/chunker/ — 7 test files, pytest", + }, +}; diff --git a/ui/chunker/src/components/WorkerPanel.tsx b/ui/chunker/src/components/WorkerPanel.tsx new file mode 100644 index 0000000..8017e10 --- /dev/null +++ b/ui/chunker/src/components/WorkerPanel.tsx @@ -0,0 +1,55 @@ +import type { WorkerInfo } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + workers: WorkerInfo[]; +} + +const STATE_COLORS: Record = { + idle: "#6b7280", + processing: "#3b82f6", + retry: "#f97316", + stopped: "#ef4444", +}; + +/** + * Worker thread status cards. + * Shows each worker's real-time state and which chunk it's processing. + * Interview Topic 2: Concurrency (threading). + */ +export function WorkerPanel({ workers }: Props) { + return ( +
+
+

Workers

+ +
+
+ {workers.map((w) => ( +
+
+ {w.worker_id} + + {w.state} + +
+ {w.current_chunk !== undefined && ( +
chunk #{w.current_chunk}
+ )} +
+ done: {w.processed} + err: {w.errors} + retry: {w.retries} +
+
+ ))} + {workers.length === 0 && ( +
No workers started
+ )} +
+
+ ); +} diff --git a/ui/chunker/src/hooks/useEventStream.ts b/ui/chunker/src/hooks/useEventStream.ts new file mode 100644 index 0000000..00c5d58 --- /dev/null +++ b/ui/chunker/src/hooks/useEventStream.ts @@ -0,0 +1,81 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import type { PipelineEvent } from "../types"; + +/** + * SSE hook — connects to /api/chunker/stream/{jobId} via native EventSource. + * + * Demonstrates: real-time event streaming from backend to UI. + */ +export function useEventStream(jobId: string | null) { + const [events, setEvents] = useState([]); + const [connected, setConnected] = useState(false); + const [done, setDone] = useState(false); + const esRef = useRef(null); + + const close = useCallback(() => { + if (esRef.current) { + esRef.current.close(); + esRef.current = null; + setConnected(false); + } + }, []); + + useEffect(() => { + if (!jobId) return; + + setEvents([]); + setDone(false); + + const es = new EventSource(`/api/chunker/stream/${jobId}`); + esRef.current = es; + + es.onopen = () => setConnected(true); + es.onerror = () => setConnected(false); + + const handleEvent = (eventType: string) => (e: MessageEvent) => { + try { + const data = JSON.parse(e.data) as PipelineEvent; + setEvents((prev) => [...prev, { ...data, status: eventType }]); + } catch { + // ignore parse errors + } + }; + + // Listen to all chunker event types + const eventTypes = [ + "waiting", + "pending", + "chunking", + "processing", + "collecting", + "completed", + "failed", + "cancelled", + "done", + "timeout", + ]; + + for (const type of eventTypes) { + es.addEventListener(type, handleEvent(type)); + } + + es.addEventListener("done", () => { + setDone(true); + es.close(); + setConnected(false); + }); + + es.addEventListener("timeout", () => { + setDone(true); + es.close(); + setConnected(false); + }); + + return () => { + es.close(); + esRef.current = null; + }; + }, [jobId]); + + return { events, connected, done, close }; +} diff --git a/ui/chunker/src/main.tsx b/ui/chunker/src/main.tsx new file mode 100644 index 0000000..e4b1b15 --- /dev/null +++ b/ui/chunker/src/main.tsx @@ -0,0 +1,9 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; +import App from "./App"; + +ReactDOM.createRoot(document.getElementById("app")!).render( + + + +); diff --git a/ui/chunker/src/types.ts b/ui/chunker/src/types.ts new file mode 100644 index 0000000..cbfd4de --- /dev/null +++ b/ui/chunker/src/types.ts @@ -0,0 +1,114 @@ +/** Pipeline configuration sent to the backend. */ +export interface PipelineConfig { + source_asset_id: string; + chunk_duration: number; + num_workers: number; + max_retries: number; + processor_type: "ffmpeg" | "checksum" | "simulated_decode" | "composite"; +} + +/** Media asset from the backend. */ +export interface MediaAsset { + id: string; + filename: string; + file_path: string; + status: string; + error_message: string | null; + file_size: number | null; + duration: number | null; + video_codec: string | null; + audio_codec: string | null; + width: number | null; + height: number | null; + framerate: number | null; + bitrate: number | null; + properties: Record; + comments: string; + tags: string[]; + created_at: string | null; + updated_at: string | null; +} + +/** State of an individual chunk. */ +export type ChunkState = + | "pending" + | "queued" + | "processing" + | "done" + | "error" + | "retry"; + +/** Tracked chunk in the UI grid. */ +export interface ChunkInfo { + sequence: number; + state: ChunkState; + size?: number; + worker_id?: string; + retries?: number; + processing_time?: number; + error?: string; +} + +/** Worker thread status. */ +export interface WorkerInfo { + worker_id: string; + state: "idle" | "processing" | "retry" | "stopped"; + current_chunk?: number; + processed: number; + errors: number; + retries: number; +} + +/** SSE event from the backend. */ +export interface PipelineEvent { + job_id: string; + status?: string; + progress?: number; + total_chunks?: number; + processed_chunks?: number; + failed_chunks?: number; + throughput_mbps?: number; + elapsed?: number; + error?: string; + // Chunk-level fields + sequence?: number; + size?: number; + worker_id?: string; + success?: boolean; + processing_time?: number; + retries?: number; + queue_size?: number; + // Worker-level fields + state?: string; + attempt?: number; + backoff?: number; +} + +/** Aggregate pipeline stats. */ +export interface PipelineStats { + total_chunks: number; + processed: number; + failed: number; + retries: number; + elapsed: number; + throughput_mbps: number; + queue_size: number; +} + +/** Error log entry. */ +export interface ErrorEntry { + timestamp: number; + sequence?: number; + worker_id?: string; + error: string; + retries?: number; + event_type: string; +} + +/** Interview topic for annotation badges. */ +export interface InterviewTopic { + number: number; + title: string; + description: string; + code_ref: string; +} diff --git a/ui/chunker/src/vite-env.d.ts b/ui/chunker/src/vite-env.d.ts new file mode 100644 index 0000000..11f02fe --- /dev/null +++ b/ui/chunker/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/ui/chunker/tsconfig.json b/ui/chunker/tsconfig.json new file mode 100644 index 0000000..d343a63 --- /dev/null +++ b/ui/chunker/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "module": "ESNext", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*.ts", "src/**/*.tsx"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/ui/chunker/tsconfig.node.json b/ui/chunker/tsconfig.node.json new file mode 100644 index 0000000..42872c5 --- /dev/null +++ b/ui/chunker/tsconfig.node.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/ui/chunker/vite.config.ts b/ui/chunker/vite.config.ts new file mode 100644 index 0000000..4e84de9 --- /dev/null +++ b/ui/chunker/vite.config.ts @@ -0,0 +1,21 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], + server: { + host: "0.0.0.0", + port: 5174, + allowedHosts: process.env.VITE_ALLOWED_HOSTS?.split(",") || [], + proxy: { + "/api": { + target: "http://fastapi:8702", + changeOrigin: true, + }, + "/graphql": { + target: "http://fastapi:8702", + changeOrigin: true, + }, + }, + }, +}); diff --git a/ui/timeline/.dockerignore b/ui/timeline/.dockerignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/ui/timeline/.dockerignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/ui/timeline/src/types.ts b/ui/timeline/src/types.ts index d739f37..891d029 100644 --- a/ui/timeline/src/types.ts +++ b/ui/timeline/src/types.ts @@ -6,6 +6,7 @@ export type AssetStatus = "pending" | "ready" | "error"; export type JobStatus = "pending" | "processing" | "completed" | "failed" | "cancelled"; +export type ChunkJobStatus = "pending" | "chunking" | "processing" | "collecting" | "completed" | "failed" | "cancelled"; export interface MediaAsset { id: string; @@ -73,6 +74,29 @@ export interface TranscodeJob { completed_at: string | null; } +export interface ChunkJob { + id: string; + source_asset_id: string; + chunk_duration: number; + num_workers: number; + max_retries: number; + processor_type: string; + status: ChunkJobStatus; + progress: number; + total_chunks: number; + processed_chunks: number; + failed_chunks: number; + retry_count: number; + error_message: string | null; + throughput_mbps: number | null; + elapsed_seconds: number | null; + celery_task_id: string | null; + priority: number; + created_at: string | null; + started_at: string | null; + completed_at: string | null; +} + export interface CreateJobRequest { source_asset_id: string; preset_id: string | null; From d5a3372d6b70cf2072c2ff596e03207303785df8 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Fri, 13 Mar 2026 14:59:32 -0300 Subject: [PATCH 2/3] ui selector --- ctrl/docker-compose.yml | 13 +++++ ctrl/landing.html | 107 +++++++++++++++++++++++++++++++++++++ ctrl/nginx.conf | 42 +++++++++++---- ui/chunker/.dockerignore | 2 + ui/chunker/Dockerfile | 12 +++++ ui/chunker/vite.config.ts | 4 ++ ui/timeline/vite.config.ts | 4 ++ 7 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 ctrl/landing.html create mode 100644 ui/chunker/.dockerignore create mode 100644 ui/chunker/Dockerfile diff --git a/ctrl/docker-compose.yml b/ctrl/docker-compose.yml index bfbf700..df2678c 100644 --- a/ctrl/docker-compose.yml +++ b/ctrl/docker-compose.yml @@ -95,10 +95,12 @@ services: - "80:80" volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ./landing.html:/etc/nginx/landing.html:ro depends_on: - django - fastapi - timeline + - chunker - minio # ============================================================================= @@ -162,6 +164,17 @@ services: volumes: - ../ui/timeline/src:/app/src + chunker: + build: + context: ../ui/chunker + dockerfile: Dockerfile + ports: + - "5174:5174" + environment: + VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-} + volumes: + - ../ui/chunker/src:/app/src + volumes: postgres-data: redis-data: diff --git a/ctrl/landing.html b/ctrl/landing.html new file mode 100644 index 0000000..7ecfafc --- /dev/null +++ b/ctrl/landing.html @@ -0,0 +1,107 @@ + + + + + + MPR + + + +
+

MPR

+

Media Processing & Review

+
+ +
▶
+
Timeline
+
Browse assets, trim, transcode
+ + +
▦
+
Chunker
+
Split media into segments, pipeline visualization
+ +
+
+ Admin + GraphQL +
+
+ + diff --git a/ctrl/nginx.conf b/ctrl/nginx.conf index 526539e..b20c6ac 100644 --- a/ctrl/nginx.conf +++ b/ctrl/nginx.conf @@ -21,6 +21,10 @@ http { server timeline:5173; } + upstream chunker { + server chunker:5174; + } + upstream minio { server minio:9000; } @@ -29,6 +33,12 @@ http { listen 80; server_name mpr.local.ar; + # Landing page + location = / { + root /etc/nginx; + try_files /landing.html =404; + } + # Django Admin location /admin { proxy_pass http://django; @@ -54,7 +64,7 @@ http { } # Timeline UI - location /ui { + location /timeline/ { proxy_pass http://timeline; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -62,8 +72,17 @@ http { proxy_set_header Connection "upgrade"; } - # Vite HMR websocket - location /@vite { + # Chunker UI + location /chunker/ { + proxy_pass http://chunker; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Vite HMR websocket (timeline) + location /timeline/@vite { proxy_pass http://timeline; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; @@ -71,6 +90,15 @@ http { proxy_set_header Host $host; } + # Vite HMR websocket (chunker) + location /chunker/@vite { + proxy_pass http://chunker; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + # Media files - proxied from MinIO (local) or S3 (AWS) location /media/in/ { proxy_pass http://minio/mpr-media-in/; @@ -81,13 +109,5 @@ http { proxy_pass http://minio/mpr-media-out/; proxy_set_header Host $http_host; } - - # Default to Timeline UI - location / { - proxy_pass http://timeline; - proxy_set_header Host $host; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } } } diff --git a/ui/chunker/.dockerignore b/ui/chunker/.dockerignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/ui/chunker/.dockerignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/ui/chunker/Dockerfile b/ui/chunker/Dockerfile new file mode 100644 index 0000000..0f0d6ab --- /dev/null +++ b/ui/chunker/Dockerfile @@ -0,0 +1,12 @@ +FROM node:20-alpine + +WORKDIR /app + +COPY package.json ./ +RUN npm install + +COPY . . + +EXPOSE 5174 + +CMD ["npm", "run", "dev"] diff --git a/ui/chunker/vite.config.ts b/ui/chunker/vite.config.ts index 4e84de9..76aa6aa 100644 --- a/ui/chunker/vite.config.ts +++ b/ui/chunker/vite.config.ts @@ -2,11 +2,15 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; export default defineConfig({ + base: "/chunker/", plugins: [react()], server: { host: "0.0.0.0", port: 5174, allowedHosts: process.env.VITE_ALLOWED_HOSTS?.split(",") || [], + hmr: { + path: "/chunker/@vite/client", + }, proxy: { "/api": { target: "http://fastapi:8702", diff --git a/ui/timeline/vite.config.ts b/ui/timeline/vite.config.ts index e38695c..39fb7ac 100644 --- a/ui/timeline/vite.config.ts +++ b/ui/timeline/vite.config.ts @@ -2,11 +2,15 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; export default defineConfig({ + base: "/timeline/", plugins: [react()], server: { host: "0.0.0.0", port: 5173, allowedHosts: process.env.VITE_ALLOWED_HOSTS?.split(",") || [], + hmr: { + path: "/timeline/@vite/client", + }, proxy: { "/api": { target: "http://fastapi:8702", From b40bd68411af15a0233ecee8e5fc113dfcb71810 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Sun, 15 Mar 2026 16:03:53 -0300 Subject: [PATCH 3/3] chunker ui redo --- core/api/chunker_sse.py | 43 +- core/api/graphql.py | 54 +- core/api/schema/graphql.py | 21 +- core/chunker/chunker.py | 23 +- core/chunker/pipeline.py | 39 +- core/chunker/worker.py | 2 + core/events.py | 40 + core/jobs/handlers/chunk.py | 66 +- core/rpc/protos/worker.proto | 22 + core/rpc/server.py | 37 + core/rpc/worker_pb2.py | 10 +- core/rpc/worker_pb2_grpc.py | 45 +- core/schema/modelgen.json | 4 +- core/schema/models/__init__.py | 16 + core/schema/models/grpc.py | 33 + core/schema/models/views.py | 57 + ctrl/docker-compose.yml | 17 +- ctrl/envoy.yaml | 64 + ctrl/generate.sh | 9 + ctrl/nginx.conf | 24 +- docs/architecture/05-chunker-pipeline.md | 290 +++ docs/architecture/index.html | 212 -- docs/architecture/styles.css | 96 +- docs/index.html | 589 +++--- modelgen/loader/schema.py | 6 + ui/chunker/package-lock.json | 120 ++ ui/chunker/package.json | 5 + ui/chunker/src/App.css | 475 ++--- ui/chunker/src/App.tsx | 169 +- ui/chunker/src/api.ts | 115 +- ui/chunker/src/components/ChunkGrid.tsx | 19 +- ui/chunker/src/components/ConfigPanel.tsx | 158 +- ui/chunker/src/components/ErrorLog.tsx | 23 - ui/chunker/src/components/OutputFiles.tsx | 51 + ui/chunker/src/components/PipelineDiagram.tsx | 50 - ui/chunker/src/components/QueueGauge.tsx | 9 +- ui/chunker/src/components/StatsPanel.tsx | 16 - ui/chunker/src/components/TopicBadge.tsx | 86 - ui/chunker/src/components/WorkerPanel.tsx | 15 +- ui/chunker/src/hooks/useEventStream.ts | 33 +- ui/chunker/src/hooks/useGrpcStream.ts | 103 + ui/chunker/src/types.ts | 76 +- ui/chunker/tsconfig.json | 9 +- ui/chunker/vite.config.ts | 25 + ui/common/api/graphql.ts | 24 + ui/common/api/grpc/worker.client.ts | 95 + ui/common/api/grpc/worker.ts | 946 +++++++++ ui/common/api/media.ts | 42 + ui/common/components/FileManager.css | 97 + ui/common/components/FileManager.tsx | 84 + ui/common/components/StatusDot.tsx | 33 + ui/common/styles/theme.css | 109 ++ ui/common/types/generated.ts | 170 ++ ui/common/utils/format.ts | 21 + ui/timeline/package-lock.json | 1736 +++++++++++++++++ ui/timeline/src/App.css | 13 +- ui/timeline/src/App.tsx | 1 - ui/timeline/src/JobPanel.tsx | 2 + ui/timeline/src/api.ts | 50 +- ui/timeline/src/types.ts | 146 +- ui/timeline/tsconfig.json | 5 +- ui/timeline/vite.config.ts | 3 + 62 files changed, 5460 insertions(+), 1493 deletions(-) create mode 100644 core/events.py create mode 100644 core/schema/models/views.py create mode 100644 ctrl/envoy.yaml create mode 100644 docs/architecture/05-chunker-pipeline.md delete mode 100644 docs/architecture/index.html create mode 100644 ui/chunker/src/components/OutputFiles.tsx delete mode 100644 ui/chunker/src/components/PipelineDiagram.tsx delete mode 100644 ui/chunker/src/components/TopicBadge.tsx create mode 100644 ui/chunker/src/hooks/useGrpcStream.ts create mode 100644 ui/common/api/graphql.ts create mode 100644 ui/common/api/grpc/worker.client.ts create mode 100644 ui/common/api/grpc/worker.ts create mode 100644 ui/common/api/media.ts create mode 100644 ui/common/components/FileManager.css create mode 100644 ui/common/components/FileManager.tsx create mode 100644 ui/common/components/StatusDot.tsx create mode 100644 ui/common/styles/theme.css create mode 100644 ui/common/types/generated.ts create mode 100644 ui/common/utils/format.ts create mode 100644 ui/timeline/package-lock.json diff --git a/core/api/chunker_sse.py b/core/api/chunker_sse.py index 5090ab1..2958e11 100644 --- a/core/api/chunker_sse.py +++ b/core/api/chunker_sse.py @@ -1,8 +1,10 @@ """ SSE endpoint for chunker pipeline events. -Bridges gRPC StreamProgress to browser-native EventSource. -GET /api/chunker/stream/{job_id} → text/event-stream +Uses Redis as the event bus between Celery workers and the SSE stream. +Celery worker pushes events via core.events, SSE endpoint polls them. + +GET /chunker/stream/{job_id} → text/event-stream """ import asyncio @@ -14,46 +16,39 @@ from typing import AsyncGenerator from fastapi import APIRouter from starlette.responses import StreamingResponse +from core.events import poll_events + logger = logging.getLogger(__name__) -router = APIRouter(prefix="/api/chunker", tags=["chunker"]) +router = APIRouter(prefix="/chunker", tags=["chunker"]) async def _event_generator(job_id: str) -> AsyncGenerator[str, None]: """ - Generate SSE events by polling gRPC job state. - - Yields server-sent events in the format: - event: - data: + Generate SSE events by polling Redis for chunk job events. """ - from core.rpc.server import _active_jobs - - last_state = None + cursor = 0 timeout = time.monotonic() + 600 # 10 min max while time.monotonic() < timeout: - job_state = _active_jobs.get(job_id) + events, cursor = poll_events(job_id, cursor) - if job_state is None: - # Job not found yet — may not have started + if not events: yield f"event: waiting\ndata: {json.dumps({'job_id': job_id})}\n\n" - await asyncio.sleep(0.5) + await asyncio.sleep(0.1) continue - # Only send if state changed - if job_state != last_state: - last_state = dict(job_state) - event_type = job_state.get("status", "update") + for data in events: + event_type = data.pop("event", "update") + payload = {**data, "job_id": job_id} - yield f"event: {event_type}\ndata: {json.dumps({**job_state, 'job_id': job_id})}\n\n" + yield f"event: {event_type}\ndata: {json.dumps(payload)}\n\n" - # End stream when job is terminal - if event_type in ("completed", "failed", "cancelled"): + if event_type in ("pipeline_complete", "pipeline_error", "cancelled"): yield f"event: done\ndata: {json.dumps({'job_id': job_id})}\n\n" - break + return - await asyncio.sleep(0.2) + await asyncio.sleep(0.05) yield f"event: timeout\ndata: {json.dumps({'job_id': job_id})}\n\n" diff --git a/core/api/graphql.py b/core/api/graphql.py index 15ea60d..6cd3b41 100644 --- a/core/api/graphql.py +++ b/core/api/graphql.py @@ -15,7 +15,9 @@ from strawberry.schema.config import StrawberryConfig from strawberry.types import Info from core.api.schema.graphql import ( + CancelResultType, ChunkJobType, + ChunkOutputFileType, CreateChunkJobInput, CreateJobInput, DeleteResultType, @@ -26,7 +28,7 @@ from core.api.schema.graphql import ( TranscodePresetType, UpdateAssetInput, ) -from core.storage import BUCKET_IN, list_objects +from core.storage import BUCKET_IN, list_objects, upload_file VIDEO_EXTS = {".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv", ".wmv", ".m4v"} AUDIO_EXTS = {".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"} @@ -90,6 +92,25 @@ class Query: def system_status(self, info: Info) -> SystemStatusType: return SystemStatusType(status="ok", version="0.1.0") + @strawberry.field + def chunk_output_files(self, info: Info, job_id: str) -> List[ChunkOutputFileType]: + """List output chunk files for a completed job from media/out/.""" + from pathlib import Path + + media_out = os.environ.get("MEDIA_OUT_DIR", "/app/media/out") + output_dir = Path(media_out) / "chunks" / job_id + if not output_dir.is_dir(): + return [] + return [ + ChunkOutputFileType( + key=f.name, + size=f.stat().st_size, + url=f"/media/out/chunks/{job_id}/{f.name}", + ) + for f in sorted(output_dir.iterdir()) + if f.is_file() + ] + # --------------------------------------------------------------------------- # Mutations @@ -100,8 +121,26 @@ class Query: class Mutation: @strawberry.mutation def scan_media_folder(self, info: Info) -> ScanResultType: + import logging + from pathlib import Path + from core.db import create_asset, get_asset_filenames + logger = logging.getLogger(__name__) + + # Sync local media/in/ files to MinIO (handles fresh installs / pruned volumes) + local_media = Path("/app/media/in") + if local_media.is_dir(): + existing_keys = {o["key"] for o in list_objects(BUCKET_IN)} + for f in local_media.iterdir(): + if f.is_file() and f.suffix.lower() in MEDIA_EXTS: + if f.name not in existing_keys: + try: + upload_file(str(f), BUCKET_IN, f.name) + logger.info("Uploaded %s to MinIO", f.name) + except Exception as e: + logger.warning("Failed to upload %s: %s", f.name, e) + objects = list_objects(BUCKET_IN, extensions=MEDIA_EXTS) existing = get_asset_filenames() @@ -284,6 +323,8 @@ class Mutation: "num_workers": input.num_workers, "max_retries": input.max_retries, "processor_type": input.processor_type, + "start_time": input.start_time, + "end_time": input.end_time, } executor_mode = os.environ.get("MPR_EXECUTOR", "local") @@ -320,6 +361,17 @@ class Mutation: celery_task_id=celery_task_id, ) + @strawberry.mutation + def cancel_chunk_job(self, info: Info, celery_task_id: str) -> CancelResultType: + """Cancel a running chunk job by revoking its Celery task.""" + try: + from admin.mpr.celery import app as celery_app + + celery_app.control.revoke(celery_task_id, terminate=True, signal="SIGTERM") + return CancelResultType(ok=True, message="Task revoked") + except Exception as e: + return CancelResultType(ok=False, message=str(e)) + # --------------------------------------------------------------------------- # Schema diff --git a/core/api/schema/graphql.py b/core/api/schema/graphql.py index 51f1bda..cf4c9a3 100644 --- a/core/api/schema/graphql.py +++ b/core/api/schema/graphql.py @@ -37,7 +37,7 @@ class MediaAssetType: file_path: Optional[str] = None status: Optional[str] = None error_message: Optional[str] = None - file_size: Optional[int] = None + file_size: Optional[float] = None duration: Optional[float] = None video_codec: Optional[str] = None audio_codec: Optional[str] = None @@ -205,3 +205,22 @@ class CreateChunkJobInput: max_retries: int = 3 processor_type: str = "ffmpeg" priority: int = 0 + start_time: Optional[float] = None + end_time: Optional[float] = None + + +@strawberry.type +class CancelResultType: + """Result of cancelling a chunk job.""" + + ok: bool = False + message: Optional[str] = None + + +@strawberry.type +class ChunkOutputFileType: + """A chunk output file in S3/MinIO with presigned download URL.""" + + key: str + size: int = 0 + url: str = "" diff --git a/core/chunker/chunker.py b/core/chunker/chunker.py index 8301c17..53a2eb0 100644 --- a/core/chunker/chunker.py +++ b/core/chunker/chunker.py @@ -28,7 +28,13 @@ class Chunker: chunk_duration: Duration of each chunk in seconds (default: 10.0) """ - def __init__(self, file_path: str, chunk_duration: float = 10.0): + def __init__( + self, + file_path: str, + chunk_duration: float = 10.0, + start_time: float | None = None, + end_time: float | None = None, + ): if not os.path.isfile(file_path): raise ChunkReadError(f"File not found: {file_path}") if chunk_duration <= 0: @@ -37,7 +43,16 @@ class Chunker: self.file_path = file_path self.chunk_duration = chunk_duration self.file_size = os.path.getsize(file_path) - self.source_duration = self._probe_duration() + full_duration = self._probe_duration() + + # Apply time range + self.range_start = max(start_time or 0.0, 0.0) + self.range_end = min(end_time or full_duration, full_duration) + if self.range_start >= self.range_end: + raise ValueError( + f"Invalid range: start={self.range_start} >= end={self.range_end}" + ) + self.source_duration = self.range_end - self.range_start def _probe_duration(self) -> float: """Get source file duration via FFmpeg probe.""" @@ -71,9 +86,9 @@ class Chunker: """ total = self.expected_chunks for sequence in range(total): - start_time = sequence * self.chunk_duration + start_time = self.range_start + sequence * self.chunk_duration end_time = min( - start_time + self.chunk_duration, self.source_duration + start_time + self.chunk_duration, self.range_end ) duration = end_time - start_time diff --git a/core/chunker/pipeline.py b/core/chunker/pipeline.py index 9bac5b8..975c249 100644 --- a/core/chunker/pipeline.py +++ b/core/chunker/pipeline.py @@ -57,6 +57,8 @@ class Pipeline: queue_size: int = 10, event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, output_dir: Optional[str] = None, + start_time: Optional[float] = None, + end_time: Optional[float] = None, ): self.source = source self.chunk_duration = chunk_duration @@ -66,6 +68,8 @@ class Pipeline: self.queue_size = queue_size self.event_callback = event_callback self.output_dir = output_dir + self.start_time = start_time + self.end_time = end_time def _emit(self, event_type: str, data: Dict[str, Any]) -> None: """Emit an event if callback is registered.""" @@ -92,6 +96,19 @@ class Pipeline: finally: chunk_queue.close() + def _monitor_progress( + self, start_time: float, file_size: int, stop_event: threading.Event + ) -> None: + """Monitor thread: emit pipeline_progress every 500ms.""" + while not stop_event.is_set(): + elapsed = time.monotonic() - start_time + mb = file_size / (1024 * 1024) + self._emit("pipeline_progress", { + "elapsed": round(elapsed, 2), + "throughput_mbps": round(mb / elapsed, 2) if elapsed > 0 else 0, + }) + stop_event.wait(0.5) + def _write_manifest( self, result: PipelineResult, source_duration: float ) -> None: @@ -146,7 +163,12 @@ class Pipeline: try: # Stage 1: Set up chunker (probes file for duration) - chunker = Chunker(self.source, self.chunk_duration) + chunker = Chunker( + self.source, + self.chunk_duration, + start_time=self.start_time, + end_time=self.end_time, + ) total_chunks = chunker.expected_chunks if total_chunks == 0: @@ -170,9 +192,18 @@ class Pipeline: output_dir=self.output_dir, ) - # Stage 3: Start workers, then produce chunks + # Stage 3: Start workers, monitor, then produce chunks pool.start() + monitor_stop = threading.Event() + monitor = threading.Thread( + target=self._monitor_progress, + args=(start_time, chunker.file_size, monitor_stop), + name="progress-monitor", + daemon=True, + ) + monitor.start() + producer = threading.Thread( target=self._produce_chunks, args=(chunker, chunk_queue), @@ -185,6 +216,10 @@ class Pipeline: all_results = pool.wait() producer.join(timeout=5.0) + # Stop monitor + monitor_stop.set() + monitor.join(timeout=2.0) + # Stage 5: Collect results in order collector = ResultCollector(total_chunks) for r in all_results: diff --git a/core/chunker/worker.py b/core/chunker/worker.py index 2fcc550..de094ca 100644 --- a/core/chunker/worker.py +++ b/core/chunker/worker.py @@ -124,6 +124,7 @@ class Worker: self._emit("chunk_processing", { "sequence": chunk.sequence, "state": "processing", + "queue_size": self.chunk_queue.qsize(), }) result = self._process_with_retry(chunk) @@ -135,6 +136,7 @@ class Worker: "success": result.success, "processing_time": result.processing_time, "retries": result.retries, + "queue_size": self.chunk_queue.qsize(), }) self._emit("worker_status", {"state": "stopped"}) diff --git a/core/events.py b/core/events.py new file mode 100644 index 0000000..5ea6980 --- /dev/null +++ b/core/events.py @@ -0,0 +1,40 @@ +""" +Redis-based event bus for pipeline job progress. + +Celery workers push events, SSE endpoints poll them. +Only depends on redis — safe to import from any context. +""" + +import json +import os + +import redis + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") + + +def _get_redis(): + return redis.from_url(REDIS_URL, decode_responses=True) + + +def push_event(job_id: str, event_type: str, data: dict) -> None: + """Push an event to the Redis list for a job.""" + r = _get_redis() + key = f"chunk_events:{job_id}" + event = json.dumps({"event": event_type, **data}) + r.rpush(key, event) + r.expire(key, 3600) + + +def poll_events(job_id: str, cursor: int = 0) -> tuple[list[dict], int]: + """Poll new events from Redis. Returns (events, new_cursor).""" + r = _get_redis() + key = f"chunk_events:{job_id}" + raw_events = r.lrange(key, cursor, -1) + parsed = [] + for raw in raw_events: + try: + parsed.append(json.loads(raw)) + except (json.JSONDecodeError, TypeError): + pass + return parsed, cursor + len(raw_events) diff --git a/core/jobs/handlers/chunk.py b/core/jobs/handlers/chunk.py index 7a06a11..352a5fe 100644 --- a/core/jobs/handlers/chunk.py +++ b/core/jobs/handlers/chunk.py @@ -2,22 +2,24 @@ ChunkHandler — job handler that wraps the chunker Pipeline. Downloads source from S3/MinIO, runs FFmpeg chunking pipeline, -uploads mp4 segments + manifest back to S3/MinIO. +writes mp4 segments + manifest to media/out/chunks/{job_id}/. +Pushes real-time events to Redis for SSE consumption. """ import logging import os -import shutil -import tempfile from typing import Any, Callable, Dict, Optional +from core.events import push_event as push_chunk_event from core.chunker import Pipeline -from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file +from core.storage import BUCKET_IN, download_to_temp from .base import Handler logger = logging.getLogger(__name__) +MEDIA_OUT_DIR = os.environ.get("MEDIA_OUT_DIR", "/app/media/out") + class ChunkHandler(Handler): """ @@ -44,14 +46,19 @@ class ChunkHandler(Handler): logger.info(f"ChunkHandler starting job {job_id}: {source_key}") # Download source from S3/MinIO + push_chunk_event(job_id, "pipeline_start", {"status": "downloading", "source_key": source_key}) tmp_source = download_to_temp(BUCKET_IN, source_key) - # Create temp output directory for chunks - tmp_output_dir = tempfile.mkdtemp(prefix=f"chunks-{job_id}-") + # Output directory: media/out/chunks/{job_id}/ + output_dir = os.path.join(MEDIA_OUT_DIR, "chunks", job_id) + if processor_type == "ffmpeg": + os.makedirs(output_dir, exist_ok=True) try: def event_bridge(event_type: str, data: Dict[str, Any]) -> None: - """Bridge pipeline events to the job progress callback.""" + """Bridge pipeline events to Redis + optional progress callback.""" + push_chunk_event(job_id, event_type, data) + if progress_callback and event_type == "pipeline_complete": progress_callback(100, data) elif progress_callback and event_type == "chunk_done": @@ -68,29 +75,28 @@ class ChunkHandler(Handler): processor_type=processor_type, queue_size=payload.get("queue_size", 10), event_callback=event_bridge, - output_dir=tmp_output_dir if processor_type == "ffmpeg" else None, + output_dir=output_dir if processor_type == "ffmpeg" else None, + start_time=payload.get("start_time"), + end_time=payload.get("end_time"), ) result = pipeline.run() - # Upload chunks + manifest to S3/MinIO + # Files are already in media/out/chunks/{job_id}/ output_prefix = f"chunks/{job_id}" - uploaded_files = [] + output_files = [ + f"{output_prefix}/{os.path.basename(f)}" + for f in result.chunk_files + ] - for chunk_file in result.chunk_files: - filename = os.path.basename(chunk_file) - output_key = f"{output_prefix}/{filename}" - upload_file(chunk_file, BUCKET_OUT, output_key) - uploaded_files.append(output_key) - logger.info(f"Uploaded {output_key}") - - # Upload manifest - manifest_path = os.path.join(tmp_output_dir, "manifest.json") - if os.path.exists(manifest_path): - manifest_key = f"{output_prefix}/manifest.json" - upload_file(manifest_path, BUCKET_OUT, manifest_key) - uploaded_files.append(manifest_key) - logger.info(f"Uploaded {manifest_key}") + push_chunk_event(job_id, "pipeline_complete", { + "status": "completed", + "total_chunks": result.total_chunks, + "processed": result.processed, + "failed": result.failed, + "elapsed": result.elapsed_time, + "throughput_mbps": result.throughput_mbps, + }) return { "status": "completed" if result.failed == 0 else "completed_with_errors", @@ -104,16 +110,16 @@ class ChunkHandler(Handler): "errors": result.errors, "chunks_in_order": result.chunks_in_order, "output_prefix": output_prefix, - "uploaded_files": uploaded_files, + "output_files": output_files, } + except Exception as e: + push_chunk_event(job_id, "pipeline_error", {"status": "failed", "error": str(e)}) + raise + finally: - # Cleanup temp files + # Cleanup temp source file only (output dir is persistent) try: os.unlink(tmp_source) except OSError: pass - try: - shutil.rmtree(tmp_output_dir, ignore_errors=True) - except OSError: - pass diff --git a/core/rpc/protos/worker.proto b/core/rpc/protos/worker.proto index 9114563..c78714a 100644 --- a/core/rpc/protos/worker.proto +++ b/core/rpc/protos/worker.proto @@ -11,6 +11,7 @@ service WorkerService { rpc StreamProgress(ProgressRequest) returns (stream ProgressUpdate); rpc CancelJob(CancelRequest) returns (CancelResponse); rpc GetWorkerStatus(Empty) returns (WorkerStatus); + rpc StreamChunkPipeline(ChunkStreamRequest) returns (stream ChunkPipelineEvent); } message JobRequest { @@ -62,3 +63,24 @@ message WorkerStatus { message Empty { // Empty } + +message ChunkStreamRequest { + string job_id = 1; +} + +message ChunkPipelineEvent { + string job_id = 1; + string event_type = 2; + int32 sequence = 3; + string worker_id = 4; + string state = 5; + int32 queue_size = 6; + float elapsed = 7; + float throughput_mbps = 8; + int32 total_chunks = 9; + int32 processed_chunks = 10; + int32 failed_chunks = 11; + string error = 12; + float processing_time = 13; + int32 retries = 14; +} diff --git a/core/rpc/server.py b/core/rpc/server.py index d07ce61..5d412da 100644 --- a/core/rpc/server.py +++ b/core/rpc/server.py @@ -173,6 +173,43 @@ class WorkerServicer(worker_pb2_grpc.WorkerServiceServicer): message="Job not found", ) + def StreamChunkPipeline(self, request, context) -> Iterator[worker_pb2.ChunkPipelineEvent]: + """Stream chunk pipeline events for a job.""" + from core.events import poll_events + + job_id = request.job_id + logger.info(f"StreamChunkPipeline: {job_id}") + + cursor = 0 + timeout = time.monotonic() + 600 # 10 min max + + while context.is_active() and time.monotonic() < timeout: + events, cursor = poll_events(job_id, cursor) + + for data in events: + event_type = data.pop("event", "") + yield worker_pb2.ChunkPipelineEvent( + job_id=job_id, + event_type=event_type, + sequence=data.get("sequence", 0), + worker_id=data.get("worker_id", ""), + state=data.get("state", ""), + queue_size=data.get("queue_size", 0), + elapsed=data.get("elapsed", 0.0), + throughput_mbps=data.get("throughput_mbps", 0.0), + total_chunks=data.get("total_chunks", 0), + processed_chunks=data.get("processed_chunks", 0), + failed_chunks=data.get("failed_chunks", 0), + error=data.get("error", ""), + processing_time=data.get("processing_time", 0.0), + retries=data.get("retries", 0), + ) + + if event_type in ("pipeline_complete", "pipeline_error"): + return + + time.sleep(0.05) + def GetWorkerStatus(self, request, context): """Get worker health and capabilities.""" try: diff --git a/core/rpc/worker_pb2.py b/core/rpc/worker_pb2.py index 80e125c..0ddf905 100644 --- a/core/rpc/worker_pb2.py +++ b/core/rpc/worker_pb2.py @@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cworker.proto\x12\nmpr.worker\"\xa7\x01\n\nJobRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x13\n\x0bsource_path\x18\x02 \x01(\t\x12\x13\n\x0boutput_path\x18\x03 \x01(\t\x12\x13\n\x0bpreset_json\x18\x04 \x01(\t\x12\x17\n\ntrim_start\x18\x05 \x01(\x02H\x00\x88\x01\x01\x12\x15\n\x08trim_end\x18\x06 \x01(\x02H\x01\x88\x01\x01\x42\r\n\x0b_trim_startB\x0b\n\t_trim_end\"@\n\x0bJobResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"!\n\x0fProgressRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9c\x01\n\x0eProgressUpdate\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x05\x12\x15\n\rcurrent_frame\x18\x03 \x01(\x05\x12\x14\n\x0c\x63urrent_time\x18\x04 \x01(\x02\x12\r\n\x05speed\x18\x05 \x01(\x02\x12\x0e\n\x06status\x18\x06 \x01(\t\x12\x12\n\x05\x65rror\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_error\"\x1f\n\rCancelRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"D\n\x0e\x43\x61ncelResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tcancelled\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x0cWorkerStatus\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x13\n\x0b\x61\x63tive_jobs\x18\x02 \x01(\x05\x12\x18\n\x10supported_codecs\x18\x03 \x03(\t\x12\x15\n\rgpu_available\x18\x04 \x01(\x08\"\x07\n\x05\x45mpty2\x9e\x02\n\rWorkerService\x12<\n\tSubmitJob\x12\x16.mpr.worker.JobRequest\x1a\x17.mpr.worker.JobResponse\x12K\n\x0eStreamProgress\x12\x1b.mpr.worker.ProgressRequest\x1a\x1a.mpr.worker.ProgressUpdate0\x01\x12\x42\n\tCancelJob\x12\x19.mpr.worker.CancelRequest\x1a\x1a.mpr.worker.CancelResponse\x12>\n\x0fGetWorkerStatus\x12\x11.mpr.worker.Empty\x1a\x18.mpr.worker.WorkerStatusb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cworker.proto\x12\nmpr.worker\"\xa7\x01\n\nJobRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x13\n\x0bsource_path\x18\x02 \x01(\t\x12\x13\n\x0boutput_path\x18\x03 \x01(\t\x12\x13\n\x0bpreset_json\x18\x04 \x01(\t\x12\x17\n\ntrim_start\x18\x05 \x01(\x02H\x00\x88\x01\x01\x12\x15\n\x08trim_end\x18\x06 \x01(\x02H\x01\x88\x01\x01\x42\r\n\x0b_trim_startB\x0b\n\t_trim_end\"@\n\x0bJobResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"!\n\x0fProgressRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9c\x01\n\x0eProgressUpdate\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x05\x12\x15\n\rcurrent_frame\x18\x03 \x01(\x05\x12\x14\n\x0c\x63urrent_time\x18\x04 \x01(\x02\x12\r\n\x05speed\x18\x05 \x01(\x02\x12\x0e\n\x06status\x18\x06 \x01(\t\x12\x12\n\x05\x65rror\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_error\"\x1f\n\rCancelRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"D\n\x0e\x43\x61ncelResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tcancelled\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x0cWorkerStatus\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x13\n\x0b\x61\x63tive_jobs\x18\x02 \x01(\x05\x12\x18\n\x10supported_codecs\x18\x03 \x03(\t\x12\x15\n\rgpu_available\x18\x04 \x01(\x08\"\x07\n\x05\x45mpty\"$\n\x12\x43hunkStreamRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\xaa\x02\n\x12\x43hunkPipelineEvent\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x12\n\nevent_type\x18\x02 \x01(\t\x12\x10\n\x08sequence\x18\x03 \x01(\x05\x12\x11\n\tworker_id\x18\x04 \x01(\t\x12\r\n\x05state\x18\x05 \x01(\t\x12\x12\n\nqueue_size\x18\x06 \x01(\x05\x12\x0f\n\x07\x65lapsed\x18\x07 \x01(\x02\x12\x17\n\x0fthroughput_mbps\x18\x08 \x01(\x02\x12\x14\n\x0ctotal_chunks\x18\t \x01(\x05\x12\x18\n\x10processed_chunks\x18\n \x01(\x05\x12\x15\n\rfailed_chunks\x18\x0b \x01(\x05\x12\r\n\x05\x65rror\x18\x0c \x01(\t\x12\x17\n\x0fprocessing_time\x18\r \x01(\x02\x12\x0f\n\x07retries\x18\x0e \x01(\x05\x32\xf7\x02\n\rWorkerService\x12<\n\tSubmitJob\x12\x16.mpr.worker.JobRequest\x1a\x17.mpr.worker.JobResponse\x12K\n\x0eStreamProgress\x12\x1b.mpr.worker.ProgressRequest\x1a\x1a.mpr.worker.ProgressUpdate0\x01\x12\x42\n\tCancelJob\x12\x19.mpr.worker.CancelRequest\x1a\x1a.mpr.worker.CancelResponse\x12>\n\x0fGetWorkerStatus\x12\x11.mpr.worker.Empty\x1a\x18.mpr.worker.WorkerStatus\x12W\n\x13StreamChunkPipeline\x12\x1e.mpr.worker.ChunkStreamRequest\x1a\x1e.mpr.worker.ChunkPipelineEvent0\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -47,6 +47,10 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['_WORKERSTATUS']._serialized_end=664 _globals['_EMPTY']._serialized_start=666 _globals['_EMPTY']._serialized_end=673 - _globals['_WORKERSERVICE']._serialized_start=676 - _globals['_WORKERSERVICE']._serialized_end=962 + _globals['_CHUNKSTREAMREQUEST']._serialized_start=675 + _globals['_CHUNKSTREAMREQUEST']._serialized_end=711 + _globals['_CHUNKPIPELINEEVENT']._serialized_start=714 + _globals['_CHUNKPIPELINEEVENT']._serialized_end=1012 + _globals['_WORKERSERVICE']._serialized_start=1015 + _globals['_WORKERSERVICE']._serialized_end=1390 # @@protoc_insertion_point(module_scope) diff --git a/core/rpc/worker_pb2_grpc.py b/core/rpc/worker_pb2_grpc.py index 402cab9..925ca59 100644 --- a/core/rpc/worker_pb2_grpc.py +++ b/core/rpc/worker_pb2_grpc.py @@ -5,7 +5,7 @@ import warnings from . import worker_pb2 as worker__pb2 -GRPC_GENERATED_VERSION = '1.76.0' +GRPC_GENERATED_VERSION = '1.78.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False @@ -54,6 +54,11 @@ class WorkerServiceStub(object): request_serializer=worker__pb2.Empty.SerializeToString, response_deserializer=worker__pb2.WorkerStatus.FromString, _registered_method=True) + self.StreamChunkPipeline = channel.unary_stream( + '/mpr.worker.WorkerService/StreamChunkPipeline', + request_serializer=worker__pb2.ChunkStreamRequest.SerializeToString, + response_deserializer=worker__pb2.ChunkPipelineEvent.FromString, + _registered_method=True) class WorkerServiceServicer(object): @@ -83,6 +88,12 @@ class WorkerServiceServicer(object): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def StreamChunkPipeline(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_WorkerServiceServicer_to_server(servicer, server): rpc_method_handlers = { @@ -106,6 +117,11 @@ def add_WorkerServiceServicer_to_server(servicer, server): request_deserializer=worker__pb2.Empty.FromString, response_serializer=worker__pb2.WorkerStatus.SerializeToString, ), + 'StreamChunkPipeline': grpc.unary_stream_rpc_method_handler( + servicer.StreamChunkPipeline, + request_deserializer=worker__pb2.ChunkStreamRequest.FromString, + response_serializer=worker__pb2.ChunkPipelineEvent.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'mpr.worker.WorkerService', rpc_method_handlers) @@ -224,3 +240,30 @@ class WorkerService(object): timeout, metadata, _registered_method=True) + + @staticmethod + def StreamChunkPipeline(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/mpr.worker.WorkerService/StreamChunkPipeline', + worker__pb2.ChunkStreamRequest.SerializeToString, + worker__pb2.ChunkPipelineEvent.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/core/schema/modelgen.json b/core/schema/modelgen.json index 80a17a9..dcf1c6d 100644 --- a/core/schema/modelgen.json +++ b/core/schema/modelgen.json @@ -13,8 +13,8 @@ }, { "target": "typescript", - "output": "ui/timeline/src/types.ts", - "include": ["dataclasses", "enums", "api"] + "output": "ui/common/types/generated.ts", + "include": ["dataclasses", "enums", "api", "views"] }, { "target": "protobuf", diff --git a/core/schema/models/__init__.py b/core/schema/models/__init__.py index b0b0dcc..d62b172 100644 --- a/core/schema/models/__init__.py +++ b/core/schema/models/__init__.py @@ -16,6 +16,8 @@ from .grpc import ( GRPC_SERVICE, CancelRequest, CancelResponse, + ChunkPipelineEvent, + ChunkStreamRequest, Empty, JobRequest, JobResponse, @@ -26,6 +28,7 @@ from .grpc import ( from .jobs import ChunkJob, ChunkJobStatus, JobStatus, TranscodeJob from .media import AssetStatus, MediaAsset from .presets import BUILTIN_PRESETS, TranscodePreset +from .views import ChunkEvent, ChunkOutputFile, PipelineStats, WorkerEvent # Core domain models - generates Django, Pydantic, TypeScript DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob, ChunkJob] @@ -44,6 +47,9 @@ API_MODELS = [ # Status enums - included in generated code ENUMS = [AssetStatus, JobStatus, ChunkJobStatus] +# View/event models - generates TypeScript for UI consumption +VIEWS = [ChunkEvent, WorkerEvent, PipelineStats, ChunkOutputFile] + # gRPC messages - generates Proto GRPC_MESSAGES = [ JobRequest, @@ -54,6 +60,8 @@ GRPC_MESSAGES = [ CancelResponse, WorkerStatus, Empty, + ChunkStreamRequest, + ChunkPipelineEvent, ] __all__ = [ @@ -82,10 +90,18 @@ __all__ = [ "CancelResponse", "WorkerStatus", "Empty", + "ChunkStreamRequest", + "ChunkPipelineEvent", + # Views + "ChunkEvent", + "WorkerEvent", + "PipelineStats", + "ChunkOutputFile", # For generator "DATACLASSES", "API_MODELS", "ENUMS", + "VIEWS", "GRPC_MESSAGES", "BUILTIN_PRESETS", ] diff --git a/core/schema/models/grpc.py b/core/schema/models/grpc.py index 313bf7b..841c0a3 100644 --- a/core/schema/models/grpc.py +++ b/core/schema/models/grpc.py @@ -41,6 +41,13 @@ class CancelRequest: job_id: str +@dataclass +class ChunkStreamRequest: + """Request to stream chunk pipeline events.""" + + job_id: str + + @dataclass class Empty: """Empty message for requests with no parameters.""" @@ -94,6 +101,26 @@ class WorkerStatus: gpu_available: bool +@dataclass +class ChunkPipelineEvent: + """Streaming chunk pipeline event.""" + + job_id: str + event_type: str # pipeline_start, chunk_queued, chunk_done, etc. + sequence: int = 0 + worker_id: str = "" + state: str = "" + queue_size: int = 0 + elapsed: float = 0.0 + throughput_mbps: float = 0.0 + total_chunks: int = 0 + processed_chunks: int = 0 + failed_chunks: int = 0 + error: str = "" + processing_time: float = 0.0 + retries: int = 0 + + # ----------------------------------------------------------------------------- # Service Definition (for documentation, generator uses this) # ----------------------------------------------------------------------------- @@ -126,5 +153,11 @@ GRPC_SERVICE = { "response": WorkerStatus, "stream_response": False, }, + { + "name": "StreamChunkPipeline", + "request": ChunkStreamRequest, + "response": ChunkPipelineEvent, + "stream_response": True, # Server streaming + }, ], } diff --git a/core/schema/models/views.py b/core/schema/models/views.py new file mode 100644 index 0000000..5d9284e --- /dev/null +++ b/core/schema/models/views.py @@ -0,0 +1,57 @@ +""" +View/Event Schema Definitions + +Projections of domain models for UI consumption via SSE events. +These reference existing schema types (e.g., ChunkJobStatus) to maintain +type-level dependencies — if the domain model changes, views update too. +""" + +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class ChunkEvent: + """SSE event for a single chunk's lifecycle.""" + + sequence: int + status: str + size: Optional[int] = None + worker_id: Optional[str] = None + processing_time: Optional[float] = None + error: Optional[str] = None + retries: int = 0 + + +@dataclass +class WorkerEvent: + """SSE event for worker state changes.""" + + worker_id: str + state: str + current_chunk: Optional[int] = None + processed: int = 0 + errors: int = 0 + retries: int = 0 + + +@dataclass +class PipelineStats: + """Aggregate pipeline statistics, updated via SSE.""" + + total_chunks: int = 0 + processed: int = 0 + failed: int = 0 + retries: int = 0 + elapsed: float = 0.0 + throughput_mbps: float = 0.0 + queue_size: int = 0 + + +@dataclass +class ChunkOutputFile: + """A chunk output file in S3/MinIO with presigned download URL.""" + + key: str + size: int = 0 + url: str = "" diff --git a/ctrl/docker-compose.yml b/ctrl/docker-compose.yml index df2678c..47c9ae2 100644 --- a/ctrl/docker-compose.yml +++ b/ctrl/docker-compose.yml @@ -89,6 +89,15 @@ services: mc anonymous set download local/mpr-media-in mc anonymous set download local/mpr-media-out + envoy: + image: envoyproxy/envoy:v1.28-latest + ports: + - "8090:8090" + volumes: + - ./envoy.yaml:/etc/envoy/envoy.yaml:ro + depends_on: + - grpc + nginx: image: nginx:alpine ports: @@ -96,12 +105,14 @@ services: volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro - ./landing.html:/etc/nginx/landing.html:ro + - ../media/out:/app/media/out:ro depends_on: - django - fastapi - timeline - chunker - minio + - envoy # ============================================================================= # Application Services @@ -139,7 +150,7 @@ services: build: context: .. dockerfile: ctrl/Dockerfile.worker - command: celery -A admin.mpr worker -l info -Q transcode -c 2 + command: celery -A admin.mpr worker -l info -Q celery,transcode -c 2 environment: <<: *common-env MPR_EXECUTOR: local @@ -163,6 +174,8 @@ services: VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-} volumes: - ../ui/timeline/src:/app/src + - ../ui/timeline/vite.config.ts:/app/vite.config.ts + - ../ui/common:/common chunker: build: @@ -174,6 +187,8 @@ services: VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-} volumes: - ../ui/chunker/src:/app/src + - ../ui/chunker/vite.config.ts:/app/vite.config.ts + - ../ui/common:/common volumes: postgres-data: diff --git a/ctrl/envoy.yaml b/ctrl/envoy.yaml new file mode 100644 index 0000000..61362e0 --- /dev/null +++ b/ctrl/envoy.yaml @@ -0,0 +1,64 @@ +admin: + address: + socket_address: { address: 0.0.0.0, port_value: 9901 } + +static_resources: + listeners: + - name: listener_0 + address: + socket_address: { address: 0.0.0.0, port_value: 8090 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: auto + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/" } + route: + cluster: grpc_service + timeout: 600s + max_stream_duration: + grpc_timeout_header_max: 600s + cors: + allow_origin_string_match: + - prefix: "*" + allow_methods: GET, PUT, DELETE, POST, OPTIONS + allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout + expose_headers: grpc-status,grpc-message + max_age: "1728000" + http_filters: + - name: envoy.filters.http.grpc_web + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb + - name: envoy.filters.http.cors + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: grpc_service + connect_timeout: 5s + type: logical_dns + lb_policy: round_robin + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: grpc_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: grpc + port_value: 50051 diff --git a/ctrl/generate.sh b/ctrl/generate.sh index f3f9337..e3dbe5f 100755 --- a/ctrl/generate.sh +++ b/ctrl/generate.sh @@ -19,4 +19,13 @@ python -m grpc_tools.protoc \ # Fix relative import in generated grpc stub sed -i 's/^import worker_pb2/from . import worker_pb2/' core/rpc/worker_pb2_grpc.py +# Generate TypeScript gRPC-Web client from proto +echo "Generating TypeScript gRPC-Web client..." +cd ui/chunker +npx protoc \ + --ts_out ../common/api/grpc \ + --proto_path ../../core/rpc/protos \ + worker.proto +cd ../.. + echo "Done!" diff --git a/ctrl/nginx.conf b/ctrl/nginx.conf index b20c6ac..1cdb4ce 100644 --- a/ctrl/nginx.conf +++ b/ctrl/nginx.conf @@ -29,6 +29,10 @@ http { server minio:9000; } + upstream envoy { + server envoy:8090; + } + server { listen 80; server_name mpr.local.ar; @@ -106,8 +110,24 @@ http { } location /media/out/ { - proxy_pass http://minio/mpr-media-out/; - proxy_set_header Host $http_host; + alias /app/media/out/; + autoindex on; + } + + # gRPC-Web proxy via Envoy + location /grpc-web/ { + proxy_pass http://envoy/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 600s; + + # Critical for streaming: disable nginx response buffering + proxy_buffering off; + proxy_cache off; + chunked_transfer_encoding on; } } } diff --git a/docs/architecture/05-chunker-pipeline.md b/docs/architecture/05-chunker-pipeline.md new file mode 100644 index 0000000..04e9e46 --- /dev/null +++ b/docs/architecture/05-chunker-pipeline.md @@ -0,0 +1,290 @@ +# Chunker Pipeline — Execution Path + +## Overview + +The chunker pipeline splits a media file into time-based segments using FFmpeg stream-copy. Events flow from worker threads through Redis and gRPC-Web streaming to the browser UI in real time. + +**7 hops from worker thread to pixel:** + +``` +Worker thread → Pipeline._emit() → event_bridge() → Redis RPUSH + → [50ms poll] gRPC server LRANGE → yield protobuf + → HTTP/2 frame → Envoy (grpc-web filter) + → HTTP/1.1 chunk → nginx (proxy_buffering off) + → fetch ReadableStream → protobuf-ts decode + → setEvents([...prev, evt]) → React re-render +``` + +--- + +## Step 1: Job Creation (Browser → GraphQL → Celery) + +``` +User clicks "Start" + → App.tsx: handleStart(config) + → api.ts: createChunkJob(config) + → POST /graphql (nginx :80 → fastapi:8702) + → graphql.py: Mutation.create_chunk_job() + → core.db: creates ChunkJob row in Postgres + → Celery: run_job.delay(job_type="chunk", job_id=..., payload=...) + → Returns { id, celery_task_id } to browser + → App.tsx: setJobId(id) — triggers gRPC stream subscription +``` + +**Files:** `ui/chunker/src/api.ts`, `core/api/graphql.py`, `core/jobs/task.py` + +--- + +## Step 2: gRPC-Web Stream (Browser → nginx → Envoy → gRPC Server) + +Once `jobId` is set, `useGrpcStream(jobId)` opens a server-streaming RPC: + +``` +useGrpcStream(jobId) fires useEffect + → GrpcWebFetchTransport({ baseUrl: "/grpc-web" }) + → WorkerServiceClient.streamChunkPipeline({ jobId }) + → fetch() POST to /grpc-web/worker.WorkerService/StreamChunkPipeline + → nginx :80 /grpc-web/ (proxy_pass → envoy:8090, proxy_buffering off) + → Envoy :8090 (grpc_web filter: HTTP/1.1 grpc-web → HTTP/2 native gRPC) + → gRPC server :50051 WorkerServicer.StreamChunkPipeline() + → Enters Redis polling loop (Step 5) +``` + +**Files:** `ui/chunker/src/hooks/useGrpcStream.ts`, `ctrl/nginx.conf`, `ctrl/envoy.yaml`, `core/rpc/server.py` + +**Key nginx config:** `proxy_buffering off` is critical — without it, nginx collects the entire upstream response before forwarding, defeating streaming entirely. + +--- + +## Step 3: Celery Worker → ChunkHandler + +``` +Celery picks up run_job task + → task.py: run_job(job_type="chunk", job_id, payload) + → registry.get_handler("chunk") → ChunkHandler + → chunk.py: ChunkHandler.process(job_id, payload) + → download_to_temp(BUCKET_IN, source_key) — pulls source from MinIO/S3 + → Creates output_dir: /app/media/out/chunks/{job_id}/ + → Constructs event_bridge callback (bridges Pipeline events → Redis) + → pipeline = Pipeline(source, ..., event_callback=event_bridge, output_dir=...) + → pipeline.run() +``` + +**Files:** `core/jobs/task.py`, `core/jobs/handlers/chunk.py` + +The `event_bridge` closure wraps every `Pipeline._emit()` call, forwarding to `push_event(job_id, event_type, data)` which writes to Redis. + +--- + +## Step 4: Pipeline Orchestration (inside Celery worker process) + +`Pipeline.run()` spawns multiple threads: + +``` +pipeline.run(): + │ + ├─ Chunker(source, chunk_duration) + │ → ffprobe source file → gets duration, file_size + │ → calculates total_chunks = ceil(duration / chunk_duration) + │ + ├─ _emit("pipeline_start", {...}) → event_bridge → Redis + ├─ _emit("pipeline_info", {file_size, duration, total_chunks}) → Redis + │ + ├─ Creates ChunkQueue(maxsize=10) + ├─ Creates WorkerPool(num_workers=N, chunk_queue, processor, event_callback) + │ + ├─ pool.start() — spawns N worker threads + │ + ├─ MONITOR THREAD starts (_monitor_progress) + │ → Every 500ms: _emit("pipeline_progress", {elapsed, throughput_mbps}) → Redis + │ + ├─ PRODUCER THREAD starts (_produce_chunks) + │ → Iterates chunker.chunks() → yields Chunk(sequence, start_time, end_time) + │ → For each: chunk_queue.put(chunk) + │ → _emit("chunk_queued", {sequence, start_time, end_time, queue_size}) → Redis + │ → chunk_queue.close() when done (sends N sentinel Nones) + │ + ├─ WORKER THREADS (N concurrent, each runs worker.py:Worker.run()) + │ │ Each worker loops: + │ │ + │ ├─ chunk = chunk_queue.get(timeout=1.0) + │ ├─ _emit("chunk_processing", {sequence, state:"processing", queue_size}) → Redis + │ │ + │ ├─ processor.process(chunk) + │ │ ├─ ffmpeg: runs `ffmpeg -ss start -to end -c copy chunk_NNNN.mp4` + │ │ ├─ simulated_decode: sleep(random) + checksum + │ │ └─ checksum: reads bytes, computes hash + │ │ + │ ├─ On success: _emit("chunk_done", {sequence, processing_time, retries, queue_size}) → Redis + │ ├─ On failure: retries with exponential backoff (0.1s, 0.2s, 0.4s...) + │ │ └─ _emit("chunk_retry", {sequence, attempt, backoff}) → Redis + │ │ └─ _emit("chunk_error", {sequence, error, retries}) → Redis (after exhaustion) + │ │ + │ └─ On sentinel (None): _emit("worker_status", {state:"stopped"}) → Redis + │ + ├─ pool.wait() — joins all worker threads, collects results + ├─ monitor_stop.set() — stops progress monitor + │ + ├─ ResultCollector — reassembles results in sequence order + │ └─ _emit("chunk_collected", {sequence, buffered, emitted}) → Redis + │ + ├─ Writes manifest.json to output_dir + │ + └─ _emit("pipeline_complete", {total_chunks, processed, failed, elapsed, throughput}) → Redis +``` + +**Files:** `core/chunker/pipeline.py`, `core/chunker/worker.py`, `core/chunker/pool.py`, `core/chunker/chunker.py`, `core/chunker/collector.py` + +--- + +## Step 5: Redis — the Event Bus + +``` +WRITE side (Celery worker, all threads): + push_event(job_id, event_type, data) + → json.dumps({"event": event_type, ...data}) + → Redis RPUSH to key "chunk_events:{job_id}" + → Redis EXPIRE 3600 (1 hour TTL) + +READ side (gRPC server, StreamChunkPipeline): + poll_events(job_id, cursor) + → Redis LRANGE "chunk_events:{job_id}" cursor -1 + → Returns (parsed_events, new_cursor) + → Called every 50ms (time.sleep(0.05) in server loop) +``` + +Redis acts as a decoupling layer between the Celery worker process (which runs the pipeline) and the gRPC server process (which streams to browsers). Events are appended with RPUSH and read with cursor-based LRANGE polling. + +**Files:** `core/events.py` + +--- + +## Step 6: gRPC Server → Envoy → nginx → Browser + +``` +server.py: StreamChunkPipeline polling loop: + while context.is_active(): + events, cursor = poll_events(job_id, cursor) ← Redis LRANGE + for data in events: + yield worker_pb2.ChunkPipelineEvent( ← serialized protobuf message + job_id, event_type, sequence, worker_id, + state, queue_size, elapsed, throughput_mbps, + total_chunks, processed_chunks, failed_chunks, + error, processing_time, retries + ) + if event_type in ("pipeline_complete", "pipeline_error"): + return ← ends the stream + time.sleep(0.05) ← 50ms poll interval + + Each yield sends: + → gRPC HTTP/2 DATA frame to Envoy + → Envoy grpc_web filter: HTTP/2 → base64-encoded grpc-web-text + → nginx proxy_pass (proxy_buffering off) → chunked HTTP/1.1 to browser + → fetch() ReadableStream in GrpcWebFetchTransport + → @protobuf-ts decodes protobuf → ChunkPipelineEvent TypeScript object +``` + +**Files:** `core/rpc/server.py`, `ctrl/envoy.yaml`, `ctrl/nginx.conf`, `ui/common/api/grpc/worker.ts`, `ui/common/api/grpc/worker.client.ts` + +--- + +## Step 7: React State Derivation and Rendering + +``` +useGrpcStream.ts: + for await (const msg of stream.responses): + const evt = toEvent(msg) ← maps protobuf camelCase → snake_case PipelineEvent + setEvents(prev => [...prev, evt]) ← appends to events array + if pipeline_complete/error → setDone(true), break + +App.tsx useMemo(events): + Iterates ALL events on every update, derives: + ├─ chunkMap: Map — state machine per chunk + │ pending → queued → processing → done/error/retry + ├─ workerMap: Map — state per worker + │ idle → processing → idle → ... → stopped + ├─ stats: PipelineStats + │ total_chunks, processed, failed, retries, elapsed, throughput_mbps, queue_size + ├─ errors: ErrorEntry[] — every event containing an error field + └─ queueSize: number — last seen queue_size value + + Renders: + ├─ ChunkGrid — colored cells per chunk (pending/queued/processing/done/error) + ├─ QueueGauge — current queue depth / max + ├─ WorkerPanel — per-worker state + current chunk assignment + ├─ StatsPanel — elapsed time, throughput, processed/failed counts + ├─ ErrorLog — scrollable error list + └─ OutputFiles — download links (when done) +``` + +**Files:** `ui/chunker/src/hooks/useGrpcStream.ts`, `ui/chunker/src/App.tsx` + +--- + +## Step 8: Output File Access (after pipeline completes) + +``` +App.tsx useEffect([done, jobId]): + → api.ts: getChunkOutputFiles(jobId) + → POST /graphql → graphql.py: chunk_output_files(job_id) + → Reads /app/media/out/chunks/{job_id}/ directory listing from disk + → Returns [{key, size, url: "/media/out/chunks/{job_id}/chunk_0001.mp4"}] + → Browser renders download links + → Click link → nginx /media/out/ → alias /app/media/out/ → serves file from disk +``` + +Chunks are written directly to `media/out/chunks/{job_id}/` by the ffmpeg processor — no MinIO upload needed for output. Nginx serves them with `autoindex on`. + +**Files:** `core/api/graphql.py`, `core/jobs/handlers/chunk.py`, `ctrl/nginx.conf` + +--- + +## Event Types Reference + +| Event | Source | Key Fields | +|-------|--------|------------| +| `pipeline_start` | Pipeline.run() | source, chunk_duration, num_workers, processor_type | +| `pipeline_info` | Pipeline.run() | file_size, source_duration, total_chunks | +| `pipeline_progress` | Monitor thread (500ms) | elapsed, throughput_mbps | +| `chunk_queued` | Producer thread | sequence, start_time, end_time, duration, queue_size | +| `chunk_processing` | Worker thread | sequence, worker_id, state, queue_size | +| `chunk_done` | Worker thread | sequence, processing_time, retries, queue_size | +| `chunk_retry` | Worker thread | sequence, attempt, backoff | +| `chunk_error` | Worker thread | sequence, error, retries | +| `chunk_collected` | ResultCollector | sequence, buffered, emitted | +| `worker_status` | Worker thread | worker_id, state (idle/processing/stopped) | +| `pipeline_complete` | Pipeline.run() | total_chunks, processed, failed, elapsed, throughput_mbps | +| `pipeline_error` | Pipeline.run() | error | + +--- + +## Thread Model (inside Celery worker) + +``` +Celery worker process + └─ run_job task thread + └─ Pipeline.run() + ├─ Producer thread — enqueues chunks + ├─ Monitor thread — emits progress every 500ms + ├─ Worker thread 0 — pulls from queue, processes + ├─ Worker thread 1 — pulls from queue, processes + ├─ Worker thread 2 — pulls from queue, processes + └─ Worker thread 3 — pulls from queue, processes +``` + +All threads share the same `event_callback` → `event_bridge` → `push_event()`, which creates a new Redis connection per call. Thread-safe via Redis atomic RPUSH. + +--- + +## Infrastructure + +| Service | Port | Role | +|---------|------|------| +| nginx | 80 | Reverse proxy, static file serving | +| fastapi | 8702 | GraphQL API (Strawberry) | +| celery | — | Task worker (runs pipeline) | +| redis | 6379 | Event bus + Celery broker | +| grpc | 50051 | gRPC server (StreamChunkPipeline) | +| envoy | 8090 | gRPC-Web ↔ native gRPC translation | +| minio | 9000 | S3-compatible source media storage | +| postgres | 5432 | Job/asset metadata | diff --git a/docs/architecture/index.html b/docs/architecture/index.html deleted file mode 100644 index e6651e7..0000000 --- a/docs/architecture/index.html +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - MPR - Architecture - - - -

MPR - Media Processor

-

- Media transcoding platform with dual execution modes: local (Celery - + MinIO) and cloud (AWS Step Functions + Lambda + S3). -

- - - -

System Overview

-
-
-

Local Architecture (Development)

- - Local Architecture - - Open full size -
-
-

AWS Architecture (Production)

- - AWS Architecture - - Open full size -
-
- -
-

Components

-
    -
  • - - Reverse Proxy (nginx) -
    • -
  • - - Application Layer (Django Admin, GraphQL API, Timeline UI) -
    • -
  • - - Worker Layer (Celery local mode) -
    • -
  • - - AWS (Step Functions, Lambda - cloud mode) -
    • -
  • - - Data Layer (PostgreSQL, Redis) -
    • -
  • - - S3 Storage (MinIO local / AWS S3 cloud) -
    • -
-
- -

Data Model

-
-
-

Entity Relationships

- - Data Model - - Open full size -
-
- -
-

Entities

-
    -
  • - - MediaAsset - Video/audio files (S3 keys as paths) -
    • -
  • - - TranscodePreset - Encoding configurations -
    • -
  • - - TranscodeJob - Processing queue (celery_task_id or - execution_arn) -
    • -
-
- -

Job Flow

-
-
-

Job Lifecycle

- - Job Flow - - Open full size -
-
- -
-

Job States

-
    -
  • - - PENDING - Waiting in queue -
    • -
  • - - PROCESSING - Worker executing -
    • -
  • - - COMPLETED - Success -
    • -
  • - - FAILED - Error occurred -
    • -
  • - - CANCELLED - User cancelled -
    • -
-

Execution Modes

-
    -
  • - - Local: Celery + MinIO (S3 API) + FFmpeg -
    • -
  • - - Lambda: Step Functions + Lambda + AWS S3 -
    • -
-
- -

Media Storage

-
-

- MPR separates media into input and output paths for flexible - storage configuration. -

-

- View Media Storage Documentation → -

-
- -

API (GraphQL)

- 
# GraphiQL IDE
-http://mpr.local.ar/graphql
-
-# Queries
-query { assets(status: "ready") { id filename duration } }
-query { jobs(status: "processing") { id status progress } }
-query { presets { id name container videoCodec } }
-query { systemStatus { status version } }
-
-# Mutations
-mutation { scanMediaFolder { found registered skipped } }
-mutation { createJob(input: { sourceAssetId: "...", presetId: "..." }) { id status } }
-mutation { cancelJob(id: "...") { id status } }
-mutation { retryJob(id: "...") { id status } }
-mutation { updateAsset(id: "...", input: { comments: "..." }) { id comments } }
-mutation { deleteAsset(id: "...") { ok } }
-
-# Lambda callback (REST)
-POST /api/jobs/{id}/callback      - Lambda completion webhook
- -

Access Points

- 
# Local development
-127.0.0.1 mpr.local.ar
-http://mpr.local.ar/admin         - Django Admin
-http://mpr.local.ar/graphql       - GraphiQL
-http://mpr.local.ar/              - Timeline UI
-http://localhost:9001              - MinIO Console
-
-# AWS deployment
-https://mpr.mcrn.ar/              - Production
- -

Quick Reference

- 
# Render SVGs from DOT files
-for f in *.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done
-
-# Switch executor mode
-MPR_EXECUTOR=local    # Celery + MinIO
-MPR_EXECUTOR=lambda   # Step Functions + Lambda + S3
- - diff --git a/docs/architecture/styles.css b/docs/architecture/styles.css index ef23579..b3094f2 100644 --- a/docs/architecture/styles.css +++ b/docs/architecture/styles.css @@ -3,6 +3,8 @@ --text-color: #e8e8e8; --accent-color: #4a90d9; --border-color: #333; + --sidebar-width: 220px; + --sidebar-bg: #151528; } * { @@ -16,6 +18,59 @@ body { background-color: var(--bg-color); color: var(--text-color); line-height: 1.6; +} + +/* Sidebar navigation */ +.sidebar { + position: fixed; + top: 0; + left: 0; + width: var(--sidebar-width); + height: 100vh; + background: var(--sidebar-bg); + border-right: 1px solid var(--border-color); + padding: 1.5rem 1rem; + overflow-y: auto; + z-index: 10; +} + +.sidebar h2 { + font-size: 1.2rem; + color: var(--accent-color); + margin-bottom: 1.5rem; + padding-bottom: 0.5rem; + border-bottom: 1px solid var(--border-color); +} + +.sidebar ul { + list-style: none; + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.sidebar li { + display: block; +} + +.sidebar a { + display: block; + padding: 0.4rem 0.6rem; + color: var(--text-color); + text-decoration: none; + font-size: 0.85rem; + border-radius: 4px; + transition: background 0.15s, color 0.15s; +} + +.sidebar a:hover { + background: rgba(74, 144, 217, 0.15); + color: var(--accent-color); +} + +/* Main content */ +.content { + margin-left: var(--sidebar-width); padding: 2rem; } @@ -25,12 +80,13 @@ h1 { color: var(--accent-color); } -h2 { +.content > h2 { font-size: 1.5rem; margin: 2rem 0 1rem; color: var(--text-color); border-bottom: 1px solid var(--border-color); padding-bottom: 0.5rem; + scroll-margin-top: 1rem; } .diagram-container { @@ -76,20 +132,6 @@ h2 { text-decoration: underline; } -nav { - margin-bottom: 2rem; -} - -nav a { - color: var(--accent-color); - text-decoration: none; - margin-right: 1.5rem; -} - -nav a:hover { - text-decoration: underline; -} - .legend { margin-top: 2rem; padding: 1rem; @@ -141,3 +183,27 @@ pre code { background: none; padding: 0; } + +/* Responsive: collapse sidebar on small screens */ +@media (max-width: 768px) { + .sidebar { + position: static; + width: 100%; + height: auto; + border-right: none; + border-bottom: 1px solid var(--border-color); + } + + .sidebar ul { + flex-direction: row; + flex-wrap: wrap; + } + + .content { + margin-left: 0; + } + + .diagram { + min-width: 100%; + } +} diff --git a/docs/index.html b/docs/index.html index 7b5f774..413af1a 100644 --- a/docs/index.html +++ b/docs/index.html @@ -7,219 +7,241 @@ -

MPR - Media Processor

-

- Media transcoding platform with three execution modes: local (Celery - + MinIO), AWS (Step Functions + Lambda + S3), and GCP (Cloud Run - Jobs + GCS). Storage is S3-compatible across all environments. -

- -
@@ -211,7 +312,10 @@ export default function App() {