From ccc478fbaa986b46af31d130ffa280a2af9e0113 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Fri, 13 Mar 2026 14:29:38 -0300 Subject: [PATCH] chunker and ui --- .dockerignore | 30 + admin/mpr/celery.py | 2 +- admin/mpr/media_assets/models.py | 40 + core/api/chunker_sse.py | 78 + core/api/graphql.py | 89 +- core/api/main.py | 4 + core/api/schema/graphql.py | 49 + core/chunker/__init__.py | 64 + core/chunker/chunker.py | 86 + core/chunker/collector.py | 98 + core/chunker/exceptions.py | 64 + core/chunker/models.py | 54 + core/chunker/pipeline.py | 244 +++ core/chunker/pool.py | 125 ++ core/chunker/processor.py | 173 ++ core/chunker/queue.py | 76 + core/chunker/worker.py | 141 ++ core/jobs/__init__.py | 15 + core/{task => jobs}/executor.py | 128 +- core/{task => jobs}/gcp_handler.py | 0 core/jobs/handlers/__init__.py | 5 + core/jobs/handlers/base.py | 33 + core/jobs/handlers/chunk.py | 119 ++ core/jobs/handlers/transcode.py | 104 + core/{task => jobs}/lambda_handler.py | 0 core/jobs/registry.py | 33 + core/jobs/task.py | 64 + core/rpc/server.py | 27 +- core/schema/models/__init__.py | 8 +- core/schema/models/jobs.py | 60 +- core/task/__init__.py | 15 - core/task/tasks.py | 105 - ctrl/Dockerfile | 3 +- ctrl/Dockerfile.worker | 3 +- ctrl/docker-compose.yml | 49 +- ctrl/lambda/Dockerfile | 6 +- tests/__init__.py | 0 tests/chunker/__init__.py | 0 tests/chunker/conftest.py | 76 + tests/chunker/test_chunker.py | 149 ++ tests/chunker/test_collector.py | 103 + tests/chunker/test_exceptions.py | 69 + tests/chunker/test_pipeline.py | 144 ++ tests/chunker/test_processor.py | 98 + tests/chunker/test_queue.py | 115 ++ tests/chunker/test_worker.py | 127 ++ ui/chunker/index.html | 12 + ui/chunker/package-lock.json | 1729 +++++++++++++++++ ui/chunker/package.json | 22 + ui/chunker/src/App.css | 735 +++++++ ui/chunker/src/App.tsx | 245 +++ ui/chunker/src/api.ts | 72 + ui/chunker/src/components/ChunkGrid.tsx | 59 + ui/chunker/src/components/ConfigPanel.tsx | 172 ++ ui/chunker/src/components/ErrorLog.tsx | 63 + ui/chunker/src/components/PipelineDiagram.tsx | 50 + ui/chunker/src/components/QueueGauge.tsx | 46 + ui/chunker/src/components/StatsPanel.tsx | 59 + ui/chunker/src/components/TopicBadge.tsx | 86 + ui/chunker/src/components/WorkerPanel.tsx | 55 + ui/chunker/src/hooks/useEventStream.ts | 81 + ui/chunker/src/main.tsx | 9 + ui/chunker/src/types.ts | 114 ++ ui/chunker/src/vite-env.d.ts | 1 + ui/chunker/tsconfig.json | 21 + ui/chunker/tsconfig.node.json | 10 + ui/chunker/vite.config.ts | 21 + ui/timeline/.dockerignore | 2 + ui/timeline/src/types.ts | 24 + 69 files changed, 6481 insertions(+), 282 deletions(-) create mode 100644 .dockerignore create mode 100644 core/api/chunker_sse.py create mode 100644 core/chunker/__init__.py create mode 100644 core/chunker/chunker.py create mode 100644 core/chunker/collector.py create mode 100644 core/chunker/exceptions.py create mode 100644 core/chunker/models.py create mode 100644 core/chunker/pipeline.py create mode 100644 core/chunker/pool.py create mode 100644 core/chunker/processor.py create mode 100644 core/chunker/queue.py create mode 100644 core/chunker/worker.py create mode 100644 core/jobs/__init__.py rename core/{task => jobs}/executor.py (53%) rename core/{task => jobs}/gcp_handler.py (100%) create mode 100644 core/jobs/handlers/__init__.py create mode 100644 core/jobs/handlers/base.py create mode 100644 core/jobs/handlers/chunk.py create mode 100644 core/jobs/handlers/transcode.py rename core/{task => jobs}/lambda_handler.py (100%) create mode 100644 core/jobs/registry.py create mode 100644 core/jobs/task.py delete mode 100644 core/task/__init__.py delete mode 100644 core/task/tasks.py create mode 100644 tests/__init__.py create mode 100644 tests/chunker/__init__.py create mode 100644 tests/chunker/conftest.py create mode 100644 tests/chunker/test_chunker.py create mode 100644 tests/chunker/test_collector.py create mode 100644 tests/chunker/test_exceptions.py create mode 100644 tests/chunker/test_pipeline.py create mode 100644 tests/chunker/test_processor.py create mode 100644 tests/chunker/test_queue.py create mode 100644 tests/chunker/test_worker.py create mode 100644 ui/chunker/index.html create mode 100644 ui/chunker/package-lock.json create mode 100644 ui/chunker/package.json create mode 100644 ui/chunker/src/App.css create mode 100644 ui/chunker/src/App.tsx create mode 100644 ui/chunker/src/api.ts create mode 100644 ui/chunker/src/components/ChunkGrid.tsx create mode 100644 ui/chunker/src/components/ConfigPanel.tsx create mode 100644 ui/chunker/src/components/ErrorLog.tsx create mode 100644 ui/chunker/src/components/PipelineDiagram.tsx create mode 100644 ui/chunker/src/components/QueueGauge.tsx create mode 100644 ui/chunker/src/components/StatsPanel.tsx create mode 100644 ui/chunker/src/components/TopicBadge.tsx create mode 100644 ui/chunker/src/components/WorkerPanel.tsx create mode 100644 ui/chunker/src/hooks/useEventStream.ts create mode 100644 ui/chunker/src/main.tsx create mode 100644 ui/chunker/src/types.ts create mode 100644 ui/chunker/src/vite-env.d.ts create mode 100644 ui/chunker/tsconfig.json create mode 100644 ui/chunker/tsconfig.node.json create mode 100644 ui/chunker/vite.config.ts create mode 100644 ui/timeline/.dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..78ee4c1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +# Python +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +.pytest_cache/ + +# Node +node_modules/ +ui/*/node_modules/ +ui/*/dist/ + +# Media (9.8GB — mounted via volume, never needed in image) +media/ + +# Git +.git/ + +# IDE / OS +.idea/ +.vscode/ +*.swp +.DS_Store + +# Docker +ctrl/docker-compose.yml + +# Docs +docs/ +*.md diff --git a/admin/mpr/celery.py b/admin/mpr/celery.py index 93c2776..358c62e 100644 --- a/admin/mpr/celery.py +++ b/admin/mpr/celery.py @@ -7,4 +7,4 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "admin.mpr.settings") app = Celery("mpr") app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() -app.autodiscover_tasks(["core.task"]) +app.autodiscover_tasks(["core.jobs"]) diff --git a/admin/mpr/media_assets/models.py b/admin/mpr/media_assets/models.py index 2ee9e29..6576093 100644 --- a/admin/mpr/media_assets/models.py +++ b/admin/mpr/media_assets/models.py @@ -19,6 +19,15 @@ class JobStatus(models.TextChoices): FAILED = "failed", "Failed" CANCELLED = "cancelled", "Cancelled" +class ChunkJobStatus(models.TextChoices): + PENDING = "pending", "Pending" + CHUNKING = "chunking", "Chunking" + PROCESSING = "processing", "Processing" + COLLECTING = "collecting", "Collecting" + COMPLETED = "completed", "Completed" + FAILED = "failed", "Failed" + CANCELLED = "cancelled", "Cancelled" + class MediaAsset(models.Model): """A video/audio file registered in the system.""" @@ -108,3 +117,34 @@ class TranscodeJob(models.Model): def __str__(self): return str(self.id) + +class ChunkJob(models.Model): + """A chunk pipeline job — splits a media file into chunks and processes them""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + source_asset_id = models.UUIDField() + chunk_duration = models.FloatField(default=10.0) + num_workers = models.IntegerField(default=4) + max_retries = models.IntegerField(default=3) + processor_type = models.CharField(max_length=255) + status = models.CharField(max_length=20, choices=ChunkJobStatus.choices, default=ChunkJobStatus.PENDING) + progress = models.FloatField(default=0.0) + total_chunks = models.IntegerField(default=0) + processed_chunks = models.IntegerField(default=0) + failed_chunks = models.IntegerField(default=0) + retry_count = models.IntegerField(default=0) + error_message = models.TextField(blank=True, default='') + throughput_mbps = models.FloatField(null=True, blank=True, default=None) + elapsed_seconds = models.FloatField(null=True, blank=True, default=None) + celery_task_id = models.CharField(max_length=255, null=True, blank=True) + priority = models.IntegerField(default=0) + created_at = models.DateTimeField(auto_now_add=True) + started_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ["-created_at"] + + def __str__(self): + return str(self.id) + diff --git a/core/api/chunker_sse.py b/core/api/chunker_sse.py new file mode 100644 index 0000000..5090ab1 --- /dev/null +++ b/core/api/chunker_sse.py @@ -0,0 +1,78 @@ +""" +SSE endpoint for chunker pipeline events. + +Bridges gRPC StreamProgress to browser-native EventSource. +GET /api/chunker/stream/{job_id} → text/event-stream +""" + +import asyncio +import json +import logging +import time +from typing import AsyncGenerator + +from fastapi import APIRouter +from starlette.responses import StreamingResponse + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/chunker", tags=["chunker"]) + + +async def _event_generator(job_id: str) -> AsyncGenerator[str, None]: + """ + Generate SSE events by polling gRPC job state. + + Yields server-sent events in the format: + event: + data: + """ + from core.rpc.server import _active_jobs + + last_state = None + timeout = time.monotonic() + 600 # 10 min max + + while time.monotonic() < timeout: + job_state = _active_jobs.get(job_id) + + if job_state is None: + # Job not found yet — may not have started + yield f"event: waiting\ndata: {json.dumps({'job_id': job_id})}\n\n" + await asyncio.sleep(0.5) + continue + + # Only send if state changed + if job_state != last_state: + last_state = dict(job_state) + event_type = job_state.get("status", "update") + + yield f"event: {event_type}\ndata: {json.dumps({**job_state, 'job_id': job_id})}\n\n" + + # End stream when job is terminal + if event_type in ("completed", "failed", "cancelled"): + yield f"event: done\ndata: {json.dumps({'job_id': job_id})}\n\n" + break + + await asyncio.sleep(0.2) + + yield f"event: timeout\ndata: {json.dumps({'job_id': job_id})}\n\n" + + +@router.get("/stream/{job_id}") +async def stream_chunk_job(job_id: str): + """ + SSE stream for a chunk pipeline job. + + The UI connects via native EventSource: + const es = new EventSource('/api/chunker/stream/'); + es.addEventListener('processing', (e) => { ... }); + """ + return StreamingResponse( + _event_generator(job_id), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) diff --git a/core/api/graphql.py b/core/api/graphql.py index 7a88e8f..15ea60d 100644 --- a/core/api/graphql.py +++ b/core/api/graphql.py @@ -15,6 +15,8 @@ from strawberry.schema.config import StrawberryConfig from strawberry.types import Info from core.api.schema.graphql import ( + ChunkJobType, + CreateChunkJobInput, CreateJobInput, DeleteResultType, MediaAssetType, @@ -172,30 +174,31 @@ class Mutation: priority=input.priority or 0, ) + payload = { + "source_key": source.file_path, + "output_key": output_filename, + "preset": preset_snapshot or None, + "trim_start": input.trim_start, + "trim_end": input.trim_end, + "duration": source.duration, + } + executor_mode = os.environ.get("MPR_EXECUTOR", "local") if executor_mode in ("lambda", "gcp"): - from core.task.executor import get_executor + from core.jobs.executor import get_executor get_executor().run( + job_type="transcode", job_id=str(job.id), - source_path=source.file_path, - output_path=output_filename, - preset=preset_snapshot or None, - trim_start=input.trim_start, - trim_end=input.trim_end, - duration=source.duration, + payload=payload, ) else: - from core.task.tasks import run_transcode_job + from core.jobs.task import run_job - result = run_transcode_job.delay( + result = run_job.delay( + job_type="transcode", job_id=str(job.id), - source_key=source.file_path, - output_key=output_filename, - preset=preset_snapshot or None, - trim_start=input.trim_start, - trim_end=input.trim_end, - duration=source.duration, + payload=payload, ) job.celery_task_id = result.id job.save(update_fields=["celery_task_id"]) @@ -261,6 +264,62 @@ class Mutation: except Exception: raise Exception("Asset not found") + @strawberry.mutation + def create_chunk_job(self, info: Info, input: CreateChunkJobInput) -> ChunkJobType: + """Create and dispatch a chunk pipeline job.""" + import uuid + + from core.db import get_asset + + try: + source = get_asset(input.source_asset_id) + except Exception: + raise Exception("Source asset not found") + + job_id = str(uuid.uuid4()) + + payload = { + "source_key": source.file_path, + "chunk_duration": input.chunk_duration, + "num_workers": input.num_workers, + "max_retries": input.max_retries, + "processor_type": input.processor_type, + } + + executor_mode = os.environ.get("MPR_EXECUTOR", "local") + celery_task_id = None + + if executor_mode in ("lambda", "gcp"): + from core.jobs.executor import get_executor + + get_executor().run( + job_type="chunk", + job_id=job_id, + payload=payload, + ) + else: + from core.jobs.task import run_job + + result = run_job.delay( + job_type="chunk", + job_id=job_id, + payload=payload, + ) + celery_task_id = result.id + + return ChunkJobType( + id=uuid.UUID(job_id), + source_asset_id=input.source_asset_id, + chunk_duration=input.chunk_duration, + num_workers=input.num_workers, + max_retries=input.max_retries, + processor_type=input.processor_type, + status="pending", + progress=0.0, + priority=input.priority, + celery_task_id=celery_task_id, + ) + # --------------------------------------------------------------------------- # Schema diff --git a/core/api/main.py b/core/api/main.py index 6844a69..8d62cf8 100644 --- a/core/api/main.py +++ b/core/api/main.py @@ -23,6 +23,7 @@ from fastapi import FastAPI, Header, HTTPException from fastapi.middleware.cors import CORSMiddleware from strawberry.fastapi import GraphQLRouter +from core.api.chunker_sse import router as chunker_router from core.api.graphql import schema as graphql_schema CALLBACK_API_KEY = os.environ.get("CALLBACK_API_KEY", "") @@ -48,6 +49,9 @@ app.add_middleware( graphql_router = GraphQLRouter(schema=graphql_schema, graphql_ide="graphiql") app.include_router(graphql_router, prefix="/graphql") +# Chunker SSE +app.include_router(chunker_router) + @app.get("/") def root(): diff --git a/core/api/schema/graphql.py b/core/api/schema/graphql.py index 48e2eb4..51f1bda 100644 --- a/core/api/schema/graphql.py +++ b/core/api/schema/graphql.py @@ -156,3 +156,52 @@ class WorkerStatusType: active_jobs: Optional[int] = None supported_codecs: Optional[List[str]] = None gpu_available: Optional[bool] = None + + +@strawberry.enum +class ChunkJobStatus(Enum): + PENDING = "pending" + CHUNKING = "chunking" + PROCESSING = "processing" + COLLECTING = "collecting" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@strawberry.type +class ChunkJobType: + """A chunk pipeline job.""" + + id: Optional[UUID] = None + source_asset_id: Optional[UUID] = None + chunk_duration: Optional[float] = None + num_workers: Optional[int] = None + max_retries: Optional[int] = None + processor_type: Optional[str] = None + status: Optional[str] = None + progress: Optional[float] = None + total_chunks: Optional[int] = None + processed_chunks: Optional[int] = None + failed_chunks: Optional[int] = None + retry_count: Optional[int] = None + error_message: Optional[str] = None + throughput_mbps: Optional[float] = None + elapsed_seconds: Optional[float] = None + celery_task_id: Optional[str] = None + priority: Optional[int] = None + created_at: Optional[datetime] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + +@strawberry.input +class CreateChunkJobInput: + """Request body for creating a chunk pipeline job.""" + + source_asset_id: UUID + chunk_duration: float = 10.0 + num_workers: int = 4 + max_retries: int = 3 + processor_type: str = "ffmpeg" + priority: int = 0 diff --git a/core/chunker/__init__.py b/core/chunker/__init__.py new file mode 100644 index 0000000..81effee --- /dev/null +++ b/core/chunker/__init__.py @@ -0,0 +1,64 @@ +""" +Chunker pipeline — splits files into chunks, processes concurrently, reassembles in order. + +Public API: + Pipeline — orchestrates the full pipeline + PipelineResult — aggregate result dataclass + Chunker — file → Chunk generator + ChunkQueue — bounded thread-safe queue + WorkerPool — manages N worker threads + ResultCollector — heapq-based ordered reassembly +""" + +from .chunker import Chunker +from .collector import ResultCollector +from .exceptions import ( + ChunkChecksumError, + ChunkError, + ChunkReadError, + PipelineError, + ProcessingError, + ProcessorFailureError, + ProcessorTimeoutError, + ReassemblyError, +) +from .models import Chunk, ChunkResult, PipelineResult +from .pipeline import Pipeline +from .pool import WorkerPool +from .processor import ( + ChecksumProcessor, + CompositeProcessor, + FFmpegExtractProcessor, + Processor, + SimulatedDecodeProcessor, +) +from .queue import ChunkQueue + +__all__ = [ + # Core + "Pipeline", + "PipelineResult", + # Components + "Chunker", + "ChunkQueue", + "WorkerPool", + "ResultCollector", + # Models + "Chunk", + "ChunkResult", + # Processors + "Processor", + "ChecksumProcessor", + "SimulatedDecodeProcessor", + "CompositeProcessor", + "FFmpegExtractProcessor", + # Exceptions + "PipelineError", + "ChunkError", + "ChunkReadError", + "ChunkChecksumError", + "ProcessingError", + "ProcessorFailureError", + "ProcessorTimeoutError", + "ReassemblyError", +] diff --git a/core/chunker/chunker.py b/core/chunker/chunker.py new file mode 100644 index 0000000..8301c17 --- /dev/null +++ b/core/chunker/chunker.py @@ -0,0 +1,86 @@ +""" +Chunker — probes a media file and yields time-based Chunk objects. + +Demonstrates: +- Function parameters and defaults (Interview Topic 1) +- List comprehensions and efficient iteration / generators (Interview Topic 3) +""" + +import math +import os +from typing import Generator + +from core.ffmpeg.probe import probe_file + +from .exceptions import ChunkReadError +from .models import Chunk + + +class Chunker: + """ + Splits a media file into time-based chunks via a generator. + + Uses FFmpeg probe to get duration, then yields Chunk objects + representing time segments (no data read — extraction happens in the processor). + + Args: + file_path: Path to the source media file + chunk_duration: Duration of each chunk in seconds (default: 10.0) + """ + + def __init__(self, file_path: str, chunk_duration: float = 10.0): + if not os.path.isfile(file_path): + raise ChunkReadError(f"File not found: {file_path}") + if chunk_duration <= 0: + raise ValueError("chunk_duration must be positive") + + self.file_path = file_path + self.chunk_duration = chunk_duration + self.file_size = os.path.getsize(file_path) + self.source_duration = self._probe_duration() + + def _probe_duration(self) -> float: + """Get source file duration via FFmpeg probe.""" + try: + result = probe_file(self.file_path) + if result.duration is None or result.duration <= 0: + raise ChunkReadError( + f"Cannot determine duration for {self.file_path}" + ) + return result.duration + except ChunkReadError: + raise + except Exception as e: + raise ChunkReadError( + f"Failed to probe {self.file_path}: {e}" + ) from e + + @property + def expected_chunks(self) -> int: + """Calculate expected number of chunks (last chunk may be shorter).""" + if self.source_duration <= 0: + return 0 + return math.ceil(self.source_duration / self.chunk_duration) + + def chunks(self) -> Generator[Chunk, None, None]: + """ + Yield Chunk objects representing time segments of the source file. + + Generator-based: chunks are yielded on demand. + Each chunk defines a time range — actual extraction is done by the processor. + """ + total = self.expected_chunks + for sequence in range(total): + start_time = sequence * self.chunk_duration + end_time = min( + start_time + self.chunk_duration, self.source_duration + ) + duration = end_time - start_time + + yield Chunk( + sequence=sequence, + start_time=start_time, + end_time=end_time, + source_path=self.file_path, + duration=duration, + ) diff --git a/core/chunker/collector.py b/core/chunker/collector.py new file mode 100644 index 0000000..4e4b5fd --- /dev/null +++ b/core/chunker/collector.py @@ -0,0 +1,98 @@ +""" +ResultCollector — reassembles chunk results in sequence order using a min-heap. + +Demonstrates: +- Algorithms and sorting (Interview Topic 6) — heapq for ordered reassembly +- Core data structures (Interview Topic 5) — heap, deque +""" + +import heapq +from collections import deque +from typing import List + +from .exceptions import ReassemblyError +from .models import ChunkResult + + +class ResultCollector: + """ + Receives ChunkResults out of order, emits them in sequence order. + + Uses a min-heap keyed on sequence number. Only emits a chunk when + all prior sequences have been accounted for. + + Args: + total_chunks: Expected total number of chunks + """ + + def __init__(self, total_chunks: int): + self.total_chunks = total_chunks + self._heap: List[tuple[int, ChunkResult]] = [] + self._next_sequence = 0 + self._emitted: List[ChunkResult] = [] + self._seen_sequences: set[int] = set() + # Sliding window for throughput calculation + self._recent_times: deque[float] = deque(maxlen=50) + + def add(self, result: ChunkResult) -> List[ChunkResult]: + """ + Add a result and return any newly emittable results in order. + + Args: + result: A ChunkResult (may arrive out of order) + + Returns: + List of results that can now be emitted in sequence order + (may be empty if we're still waiting for earlier sequences) + + Raises: + ReassemblyError: If a duplicate sequence is received + """ + if result.sequence in self._seen_sequences: + raise ReassemblyError( + f"Duplicate sequence number: {result.sequence}" + ) + self._seen_sequences.add(result.sequence) + + # Track processing time for throughput + if result.processing_time > 0: + self._recent_times.append(result.processing_time) + + # Push to min-heap + heapq.heappush(self._heap, (result.sequence, result)) + + # Emit all consecutive results starting from _next_sequence + newly_emitted = [] + while self._heap and self._heap[0][0] == self._next_sequence: + _, emitted_result = heapq.heappop(self._heap) + self._emitted.append(emitted_result) + newly_emitted.append(emitted_result) + self._next_sequence += 1 + + return newly_emitted + + @property + def is_complete(self) -> bool: + """True if all expected chunks have been emitted in order.""" + return self._next_sequence == self.total_chunks + + @property + def buffered_count(self) -> int: + """Number of results waiting in the heap (arrived out of order).""" + return len(self._heap) + + @property + def emitted_count(self) -> int: + """Number of results emitted in sequence order.""" + return len(self._emitted) + + @property + def avg_processing_time(self) -> float: + """Average processing time from recent results (sliding window).""" + if not self._recent_times: + return 0.0 + return sum(self._recent_times) / len(self._recent_times) + + def get_ordered_results(self) -> List[ChunkResult]: + """Get all emitted results in sequence order.""" + return list(self._emitted) diff --git a/core/chunker/exceptions.py b/core/chunker/exceptions.py new file mode 100644 index 0000000..2426152 --- /dev/null +++ b/core/chunker/exceptions.py @@ -0,0 +1,64 @@ +""" +Chunker exception hierarchy. + +Demonstrates: Managing exceptions and writing resilient code (Interview Topic 7). +""" + + +class PipelineError(Exception): + """Base exception for all chunker pipeline errors.""" + pass + + +class ChunkError(PipelineError): + """Errors related to chunk creation or validation.""" + pass + + +class ChunkReadError(ChunkError): + """Failed to read chunk data from source file.""" + pass + + +class ChunkChecksumError(ChunkError): + """Chunk data integrity validation failed.""" + + def __init__(self, sequence: int, expected: str, actual: str): + self.sequence = sequence + self.expected = expected + self.actual = actual + super().__init__( + f"Chunk {sequence}: checksum mismatch " + f"(expected={expected}, actual={actual})" + ) + + +class ProcessingError(PipelineError): + """Errors during chunk processing by workers.""" + pass + + +class ProcessorTimeoutError(ProcessingError): + """Processor exceeded allowed time for a chunk.""" + + def __init__(self, sequence: int, timeout: float): + self.sequence = sequence + self.timeout = timeout + super().__init__(f"Chunk {sequence}: processor timed out after {timeout}s") + + +class ProcessorFailureError(ProcessingError): + """Processor failed to process a chunk after all retries.""" + + def __init__(self, sequence: int, retries: int, original_error: Exception): + self.sequence = sequence + self.retries = retries + self.original_error = original_error + super().__init__( + f"Chunk {sequence}: failed after {retries} retries — {original_error}" + ) + + +class ReassemblyError(PipelineError): + """Errors during result collection and ordering.""" + pass diff --git a/core/chunker/models.py b/core/chunker/models.py new file mode 100644 index 0000000..d2f6a7d --- /dev/null +++ b/core/chunker/models.py @@ -0,0 +1,54 @@ +""" +Internal data models for the chunker pipeline. + +These are pipeline-internal dataclasses, not schema models. +Schema-level ChunkJob is in core/schema/models/jobs.py. + +Demonstrates: Core data structures (Interview Topic 5). +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class Chunk: + """A time-based segment of the source media file.""" + + sequence: int + start_time: float # seconds + end_time: float # seconds + source_path: str # path to source file + duration: float # end_time - start_time + checksum: str = "" # computed after extraction + + +@dataclass +class ChunkResult: + """Result of processing a single chunk.""" + + sequence: int + success: bool + checksum_valid: bool = True + processing_time: float = 0.0 + error: Optional[str] = None + retries: int = 0 + worker_id: Optional[str] = None + output_file: Optional[str] = None + + +@dataclass +class PipelineResult: + """Aggregate result of the entire pipeline run.""" + + total_chunks: int = 0 + processed: int = 0 + failed: int = 0 + retries: int = 0 + elapsed_time: float = 0.0 + throughput_mbps: float = 0.0 + worker_stats: Dict[str, Any] = field(default_factory=dict) + errors: List[str] = field(default_factory=list) + chunks_in_order: bool = True + output_dir: Optional[str] = None + chunk_files: List[str] = field(default_factory=list) diff --git a/core/chunker/pipeline.py b/core/chunker/pipeline.py new file mode 100644 index 0000000..9bac5b8 --- /dev/null +++ b/core/chunker/pipeline.py @@ -0,0 +1,244 @@ +""" +Pipeline — orchestrates the entire chunker pipeline. + +Wires: Chunker → ChunkQueue → WorkerPool → ResultCollector → PipelineResult + +Demonstrates: +- Function parameters and defaults (Interview Topic 1) — configurable pipeline +- Concurrency (Interview Topic 2) — producer thread + worker pool +- OOP design (Interview Topic 4) — composition of pipeline components +- Exception handling (Interview Topic 7) — graceful error propagation +""" + +import json +import logging +import threading +import time +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +from .chunker import Chunker +from .collector import ResultCollector +from .exceptions import PipelineError +from .models import PipelineResult +from .pool import WorkerPool +from .queue import ChunkQueue + +logger = logging.getLogger(__name__) + + +class Pipeline: + """ + Orchestrates the chunk processing pipeline. + + The pipeline runs in three stages: + 1. Producer thread: Chunker probes file → pushes time-based chunks to ChunkQueue + 2. Worker pool: N workers pull from queue → extract mp4 segments → emit results + 3. Collector: ResultCollector reassembles results in sequence order + + Args: + source: Path to the source media file + chunk_duration: Duration of each chunk in seconds (default: 10.0) + num_workers: Number of concurrent worker threads (default: 4) + max_retries: Max retry attempts per chunk (default: 3) + processor_type: Processor to use — "ffmpeg", "checksum", "simulated_decode", "composite" + queue_size: Max chunks buffered in queue (default: 10) + event_callback: Optional callback for real-time events + output_dir: Directory for output chunk files (required for "ffmpeg" processor) + """ + + def __init__( + self, + source: str, + chunk_duration: float = 10.0, + num_workers: int = 4, + max_retries: int = 3, + processor_type: str = "checksum", + queue_size: int = 10, + event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + output_dir: Optional[str] = None, + ): + self.source = source + self.chunk_duration = chunk_duration + self.num_workers = num_workers + self.max_retries = max_retries + self.processor_type = processor_type + self.queue_size = queue_size + self.event_callback = event_callback + self.output_dir = output_dir + + def _emit(self, event_type: str, data: Dict[str, Any]) -> None: + """Emit an event if callback is registered.""" + if self.event_callback: + self.event_callback(event_type, data) + + def _produce_chunks( + self, chunker: Chunker, chunk_queue: ChunkQueue + ) -> None: + """Producer thread: probe file and enqueue time-based chunks.""" + try: + for chunk in chunker.chunks(): + chunk_queue.put(chunk, timeout=30.0) + self._emit("chunk_queued", { + "sequence": chunk.sequence, + "start_time": chunk.start_time, + "end_time": chunk.end_time, + "duration": chunk.duration, + "queue_size": chunk_queue.qsize(), + }) + except Exception as e: + logger.error(f"Producer error: {e}") + self._emit("producer_error", {"error": str(e)}) + finally: + chunk_queue.close() + + def _write_manifest( + self, result: PipelineResult, source_duration: float + ) -> None: + """Write manifest.json to output_dir with segment metadata.""" + if not self.output_dir: + return + + manifest = { + "source": self.source, + "source_duration": source_duration, + "chunk_duration": self.chunk_duration, + "total_chunks": result.total_chunks, + "processed": result.processed, + "failed": result.failed, + "elapsed_time": result.elapsed_time, + "throughput_mbps": result.throughput_mbps, + "segments": [ + { + "sequence": i, + "file": f"chunk_{i:04d}.mp4", + "start": i * self.chunk_duration, + "end": min( + (i + 1) * self.chunk_duration, source_duration + ), + } + for i in range(result.total_chunks) + if i < result.total_chunks + ], + } + + manifest_path = Path(self.output_dir) / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2)) + logger.info(f"Manifest written to {manifest_path}") + + def run(self) -> PipelineResult: + """ + Execute the full pipeline. + + Returns: + PipelineResult with aggregate stats + + Raises: + PipelineError: If the pipeline fails catastrophically + """ + start_time = time.monotonic() + self._emit("pipeline_start", { + "source": self.source, + "chunk_duration": self.chunk_duration, + "num_workers": self.num_workers, + "processor_type": self.processor_type, + }) + + try: + # Stage 1: Set up chunker (probes file for duration) + chunker = Chunker(self.source, self.chunk_duration) + total_chunks = chunker.expected_chunks + + if total_chunks == 0: + self._emit("pipeline_complete", {"total_chunks": 0}) + return PipelineResult(chunks_in_order=True) + + self._emit("pipeline_info", { + "file_size": chunker.file_size, + "source_duration": chunker.source_duration, + "total_chunks": total_chunks, + }) + + # Stage 2: Set up queue and worker pool + chunk_queue = ChunkQueue(maxsize=self.queue_size) + pool = WorkerPool( + num_workers=self.num_workers, + chunk_queue=chunk_queue, + processor_type=self.processor_type, + max_retries=self.max_retries, + event_callback=self.event_callback, + output_dir=self.output_dir, + ) + + # Stage 3: Start workers, then produce chunks + pool.start() + + producer = threading.Thread( + target=self._produce_chunks, + args=(chunker, chunk_queue), + name="chunk-producer", + daemon=True, + ) + producer.start() + + # Stage 4: Wait for all workers to finish + all_results = pool.wait() + producer.join(timeout=5.0) + + # Stage 5: Collect results in order + collector = ResultCollector(total_chunks) + for r in all_results: + collector.add(r) + self._emit("chunk_collected", { + "sequence": r.sequence, + "success": r.success, + "buffered": collector.buffered_count, + "emitted": collector.emitted_count, + }) + + # Build result + elapsed = time.monotonic() - start_time + file_size_mb = chunker.file_size / (1024 * 1024) + throughput = file_size_mb / elapsed if elapsed > 0 else 0.0 + + failed_results = [r for r in all_results if not r.success] + total_retries = sum(r.retries for r in all_results) + chunk_files = [ + r.output_file for r in all_results + if r.success and r.output_file + ] + + result = PipelineResult( + total_chunks=total_chunks, + processed=len(all_results), + failed=len(failed_results), + retries=total_retries, + elapsed_time=elapsed, + throughput_mbps=throughput, + worker_stats=pool.get_worker_stats(), + errors=[r.error for r in failed_results if r.error], + chunks_in_order=collector.is_complete, + output_dir=self.output_dir, + chunk_files=chunk_files, + ) + + # Write manifest if output_dir is set + self._write_manifest(result, chunker.source_duration) + + pool.shutdown() + + self._emit("pipeline_complete", { + "total_chunks": result.total_chunks, + "processed": result.processed, + "failed": result.failed, + "elapsed": result.elapsed_time, + "throughput_mbps": result.throughput_mbps, + }) + + return result + + except PipelineError: + raise + except Exception as e: + self._emit("pipeline_error", {"error": str(e)}) + raise PipelineError(f"Pipeline failed: {e}") from e diff --git a/core/chunker/pool.py b/core/chunker/pool.py new file mode 100644 index 0000000..bc86d04 --- /dev/null +++ b/core/chunker/pool.py @@ -0,0 +1,125 @@ +""" +WorkerPool — manages N worker threads via ThreadPoolExecutor. + +Demonstrates: Python concurrency — threading (Interview Topic 2). +""" + +import logging +import threading +from concurrent.futures import Future, ThreadPoolExecutor +from typing import Any, Callable, Dict, List, Optional + +from .models import ChunkResult +from .processor import ( + ChecksumProcessor, + CompositeProcessor, + FFmpegExtractProcessor, + Processor, + SimulatedDecodeProcessor, +) +from .queue import ChunkQueue +from .worker import Worker + +logger = logging.getLogger(__name__) + + +def create_processor( + processor_type: str = "checksum", + output_dir: Optional[str] = None, +) -> Processor: + """Factory for processor instances.""" + if processor_type == "ffmpeg": + if not output_dir: + raise ValueError("output_dir required for ffmpeg processor") + return FFmpegExtractProcessor(output_dir=output_dir) + elif processor_type == "checksum": + return ChecksumProcessor() + elif processor_type == "simulated_decode": + return SimulatedDecodeProcessor() + elif processor_type == "composite": + return CompositeProcessor([ + ChecksumProcessor(), + SimulatedDecodeProcessor(ms_per_second=50.0), + ]) + else: + raise ValueError(f"Unknown processor type: {processor_type}") + + +class WorkerPool: + """ + Manages N worker threads that process chunks concurrently. + + Args: + num_workers: Number of concurrent worker threads (default: 4) + chunk_queue: Shared queue to pull chunks from + processor_type: Type of processor for each worker (default: "checksum") + max_retries: Max retry attempts per chunk (default: 3) + event_callback: Optional callback for real-time events + """ + + def __init__( + self, + num_workers: int = 4, + chunk_queue: Optional[ChunkQueue] = None, + processor_type: str = "checksum", + max_retries: int = 3, + event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + output_dir: Optional[str] = None, + ): + self.num_workers = num_workers + self.chunk_queue = chunk_queue or ChunkQueue() + self.processor_type = processor_type + self.max_retries = max_retries + self.event_callback = event_callback + self.output_dir = output_dir + self.shutdown_event = threading.Event() + self._executor: Optional[ThreadPoolExecutor] = None + self._futures: List[Future] = [] + self._workers: List[Worker] = [] + + def start(self) -> None: + """Start all worker threads.""" + self._executor = ThreadPoolExecutor( + max_workers=self.num_workers, + thread_name_prefix="chunk-worker", + ) + + for i in range(self.num_workers): + worker = Worker( + worker_id=f"worker-{i}", + chunk_queue=self.chunk_queue, + processor=create_processor(self.processor_type, output_dir=self.output_dir), + max_retries=self.max_retries, + event_callback=self.event_callback, + ) + self._workers.append(worker) + future = self._executor.submit(worker.run) + self._futures.append(future) + + logger.info(f"WorkerPool started with {self.num_workers} workers") + + def wait(self) -> List[ChunkResult]: + """Wait for all workers to finish and collect results.""" + all_results = [] + for future in self._futures: + results = future.result() + all_results.extend(results) + return all_results + + def shutdown(self) -> None: + """Signal shutdown and cleanup.""" + self.shutdown_event.set() + self.chunk_queue.close() + if self._executor: + self._executor.shutdown(wait=True) + + def get_worker_stats(self) -> Dict[str, Any]: + """Get per-worker statistics.""" + return { + w.worker_id: { + "processed": w.processed_count, + "errors": w.error_count, + "retries": w.retry_count, + } + for w in self._workers + } diff --git a/core/chunker/processor.py b/core/chunker/processor.py new file mode 100644 index 0000000..dd5d772 --- /dev/null +++ b/core/chunker/processor.py @@ -0,0 +1,173 @@ +""" +Processor ABC and concrete implementations. + +Demonstrates: OOP design principles — ABC, inheritance, composition (Interview Topic 4). +""" + +import hashlib +import time +from abc import ABC, abstractmethod +from pathlib import Path +from typing import List + +from .exceptions import ChunkChecksumError +from .models import Chunk, ChunkResult + + +class Processor(ABC): + """ + Abstract base class for chunk processors. + + Each processor defines how a single chunk is processed. + The Worker calls processor.process(chunk) and handles retries. + """ + + @abstractmethod + def process(self, chunk: Chunk) -> ChunkResult: + """Process a single chunk and return the result.""" + pass + + +class FFmpegExtractProcessor(Processor): + """ + Extracts a time segment from the source file using FFmpeg stream copy. + + Produces a playable mp4 file per chunk — no re-encoding. + + Args: + output_dir: Directory to write chunk mp4 files + """ + + def __init__(self, output_dir: str): + self.output_dir = output_dir + Path(output_dir).mkdir(parents=True, exist_ok=True) + + def process(self, chunk: Chunk) -> ChunkResult: + from core.ffmpeg.transcode import TranscodeConfig, transcode + + start = time.monotonic() + + output_file = str( + Path(self.output_dir) / f"chunk_{chunk.sequence:04d}.mp4" + ) + + config = TranscodeConfig( + input_path=chunk.source_path, + output_path=output_file, + video_codec="copy", + audio_codec="copy", + trim_start=chunk.start_time, + trim_end=chunk.end_time, + ) + + transcode(config) + + # Compute checksum of output file + md5 = hashlib.md5() + with open(output_file, "rb") as f: + for block in iter(lambda: f.read(8192), b""): + md5.update(block) + checksum = md5.hexdigest() + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=True, + processing_time=elapsed, + output_file=output_file, + ) + + +class ChecksumProcessor(Processor): + """ + Validates chunk metadata consistency. + + For time-based chunks, verifies the time range is valid. + Raises ChunkChecksumError on invalid ranges. + """ + + def process(self, chunk: Chunk) -> ChunkResult: + start = time.monotonic() + + valid = chunk.duration > 0 and chunk.end_time > chunk.start_time + + if not valid: + raise ChunkChecksumError( + sequence=chunk.sequence, + expected="valid time range", + actual=f"{chunk.start_time}-{chunk.end_time}", + ) + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=True, + processing_time=elapsed, + ) + + +class SimulatedDecodeProcessor(Processor): + """ + Simulates decode work by sleeping proportional to chunk duration. + + Useful for demonstrating concurrency behavior without real FFmpeg. + + Args: + ms_per_second: Milliseconds of simulated work per second of chunk duration (default: 100) + """ + + def __init__(self, ms_per_second: float = 100.0): + self.ms_per_second = ms_per_second + + def process(self, chunk: Chunk) -> ChunkResult: + start = time.monotonic() + + sleep_time = (self.ms_per_second * chunk.duration) / 1000.0 + time.sleep(sleep_time) + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=True, + processing_time=elapsed, + ) + + +class CompositeProcessor(Processor): + """ + Chains multiple processors — runs each in sequence on the same chunk. + + Demonstrates OOP composition pattern. + + Args: + processors: List of processors to chain + """ + + def __init__(self, processors: List[Processor]): + if not processors: + raise ValueError("CompositeProcessor requires at least one processor") + self.processors = processors + + def process(self, chunk: Chunk) -> ChunkResult: + start = time.monotonic() + last_result = None + + for proc in self.processors: + last_result = proc.process(chunk) + if not last_result.success: + return last_result + + elapsed = time.monotonic() - start + + return ChunkResult( + sequence=chunk.sequence, + success=True, + checksum_valid=last_result.checksum_valid if last_result else True, + processing_time=elapsed, + ) diff --git a/core/chunker/queue.py b/core/chunker/queue.py new file mode 100644 index 0000000..191a219 --- /dev/null +++ b/core/chunker/queue.py @@ -0,0 +1,76 @@ +""" +ChunkQueue — bounded, thread-safe queue with sentinel-based shutdown. + +Demonstrates: Core data structures — queue.Queue (Interview Topic 5). +""" + +import queue +from typing import Optional + +from .models import Chunk + +# Sentinel value to signal workers to stop +_SENTINEL = object() + + +class ChunkQueue: + """ + Thread-safe bounded queue for chunks. + + Provides backpressure: producers block when the queue is full, + preventing unbounded memory usage. + + Args: + maxsize: Maximum number of chunks in the queue (default: 10) + """ + + def __init__(self, maxsize: int = 10): + self._queue: queue.Queue = queue.Queue(maxsize=maxsize) + self._closed = False + self.maxsize = maxsize + + def put(self, chunk: Chunk, timeout: Optional[float] = None) -> None: + """ + Add a chunk to the queue. Blocks if full (backpressure). + + Args: + chunk: The chunk to enqueue + timeout: Max seconds to wait (None = block forever) + + Raises: + queue.Full: If timeout expires while queue is full + """ + self._queue.put(chunk, timeout=timeout) + + def get(self, timeout: Optional[float] = None) -> Optional[Chunk]: + """ + Get next chunk from queue. Returns None if queue is closed. + + Args: + timeout: Max seconds to wait (None = block forever) + + Returns: + Chunk or None (if sentinel received, meaning queue is closed) + + Raises: + queue.Empty: If timeout expires while queue is empty + """ + item = self._queue.get(timeout=timeout) + if item is _SENTINEL: + # Re-put sentinel so other workers also see it + self._queue.put(_SENTINEL) + return None + return item + + def close(self) -> None: + """Signal all consumers to stop by inserting a sentinel.""" + self._closed = True + self._queue.put(_SENTINEL) + + @property + def is_closed(self) -> bool: + return self._closed + + def qsize(self) -> int: + """Current number of items in the queue (approximate).""" + return self._queue.qsize() diff --git a/core/chunker/worker.py b/core/chunker/worker.py new file mode 100644 index 0000000..2fcc550 --- /dev/null +++ b/core/chunker/worker.py @@ -0,0 +1,141 @@ +""" +Worker — pulls chunks from queue, processes with retry logic. + +Demonstrates: +- Exception handling and resilient code (Interview Topic 7) +- Concurrency (Interview Topic 2) — workers run in thread pool +""" + +import logging +import queue +import time +from typing import Any, Callable, Dict, Optional + +from .exceptions import ProcessorFailureError +from .models import Chunk, ChunkResult +from .processor import Processor +from .queue import ChunkQueue + +logger = logging.getLogger(__name__) + + +class Worker: + """ + Processes chunks from a queue with retry and exponential backoff. + + Args: + worker_id: Identifier for this worker (e.g. "worker-0") + chunk_queue: Source queue to pull chunks from + processor: Processor instance to use + max_retries: Maximum retry attempts per chunk (default: 3) + event_callback: Optional callback for real-time status updates + """ + + def __init__( + self, + worker_id: str, + chunk_queue: ChunkQueue, + processor: Processor, + max_retries: int = 3, + event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + ): + self.worker_id = worker_id + self.chunk_queue = chunk_queue + self.processor = processor + self.max_retries = max_retries + self.event_callback = event_callback + self.processed_count = 0 + self.error_count = 0 + self.retry_count = 0 + + def _emit(self, event_type: str, data: Dict[str, Any]) -> None: + """Emit an event if callback is registered.""" + if self.event_callback: + self.event_callback(event_type, {"worker_id": self.worker_id, **data}) + + def _process_with_retry(self, chunk: Chunk) -> ChunkResult: + """ + Process a chunk with exponential backoff retry. + + Retry delays: 0.1s, 0.2s, 0.4s, ... (doubles each attempt) + """ + last_error = None + + for attempt in range(self.max_retries + 1): + try: + if attempt > 0: + backoff = 0.1 * (2 ** (attempt - 1)) + self._emit("chunk_retry", { + "sequence": chunk.sequence, + "attempt": attempt, + "backoff": backoff, + }) + time.sleep(backoff) + self.retry_count += 1 + + result = self.processor.process(chunk) + result.retries = attempt + result.worker_id = self.worker_id + return result + + except Exception as e: + last_error = e + logger.warning( + f"{self.worker_id}: chunk {chunk.sequence} " + f"attempt {attempt + 1}/{self.max_retries + 1} failed: {e}" + ) + + # All retries exhausted + self.error_count += 1 + self._emit("chunk_error", { + "sequence": chunk.sequence, + "error": str(last_error), + "retries": self.max_retries, + }) + + return ChunkResult( + sequence=chunk.sequence, + success=False, + processing_time=0.0, + error=str(last_error), + retries=self.max_retries, + worker_id=self.worker_id, + ) + + def run(self) -> list[ChunkResult]: + """ + Main worker loop — pull chunks and process until queue is closed. + + Returns: + List of ChunkResults processed by this worker + """ + results = [] + self._emit("worker_status", {"state": "idle"}) + + while True: + try: + chunk = self.chunk_queue.get(timeout=1.0) + except queue.Empty: + continue + + if chunk is None: # Sentinel received + break + + self._emit("chunk_processing", { + "sequence": chunk.sequence, + "state": "processing", + }) + + result = self._process_with_retry(chunk) + results.append(result) + self.processed_count += 1 + + self._emit("chunk_done", { + "sequence": chunk.sequence, + "success": result.success, + "processing_time": result.processing_time, + "retries": result.retries, + }) + + self._emit("worker_status", {"state": "stopped"}) + return results diff --git a/core/jobs/__init__.py b/core/jobs/__init__.py new file mode 100644 index 0000000..8827db9 --- /dev/null +++ b/core/jobs/__init__.py @@ -0,0 +1,15 @@ +""" +MPR Jobs Module + +Provides executor abstraction and task dispatch for job processing. +""" + +from .executor import Executor, LocalExecutor, get_executor +from .task import run_job + +__all__ = [ + "Executor", + "LocalExecutor", + "get_executor", + "run_job", +] diff --git a/core/task/executor.py b/core/jobs/executor.py similarity index 53% rename from core/task/executor.py rename to core/jobs/executor.py index aadf783..ef1e6cd 100644 --- a/core/task/executor.py +++ b/core/jobs/executor.py @@ -1,17 +1,16 @@ """ Executor abstraction for job processing. -Supports different backends: -- LocalExecutor: FFmpeg via Celery (default) -- LambdaExecutor: AWS Lambda (future) +Determines WHERE jobs run: +- LocalExecutor: delegates to registered Handler (default) +- LambdaExecutor: AWS Step Functions +- GCPExecutor: Google Cloud Run Jobs """ import os from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Optional -from core.ffmpeg.transcode import TranscodeConfig, transcode - # Configuration from environment MPR_EXECUTOR = os.environ.get("MPR_EXECUTOR", "local") @@ -22,26 +21,18 @@ class Executor(ABC): @abstractmethod def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: """ - Execute a transcode/trim job. + Execute a job. Args: + job_type: Type of job ("transcode", "chunk", etc.) job_id: Unique job identifier - source_path: Path to source file - output_path: Path for output file - preset: Transcode preset dict (optional, None = trim only) - trim_start: Trim start time in seconds (optional) - trim_end: Trim end time in seconds (optional) - duration: Source duration in seconds (for progress calculation) + payload: Job-type-specific configuration dict progress_callback: Called with (percent, details_dict) Returns: @@ -51,62 +42,25 @@ class Executor(ABC): class LocalExecutor(Executor): - """Execute jobs locally using FFmpeg.""" + """Execute jobs locally using registered handlers.""" def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: - """Execute job using local FFmpeg.""" + """Execute job using the appropriate local handler.""" + from .registry import get_handler - # Build config from preset or use stream copy for trim-only - if preset: - config = TranscodeConfig( - input_path=source_path, - output_path=output_path, - video_codec=preset.get("video_codec", "libx264"), - video_bitrate=preset.get("video_bitrate"), - video_crf=preset.get("video_crf"), - video_preset=preset.get("video_preset"), - resolution=preset.get("resolution"), - framerate=preset.get("framerate"), - audio_codec=preset.get("audio_codec", "aac"), - audio_bitrate=preset.get("audio_bitrate"), - audio_channels=preset.get("audio_channels"), - audio_samplerate=preset.get("audio_samplerate"), - container=preset.get("container", "mp4"), - extra_args=preset.get("extra_args", []), - trim_start=trim_start, - trim_end=trim_end, - ) - else: - # Trim-only: stream copy - config = TranscodeConfig( - input_path=source_path, - output_path=output_path, - video_codec="copy", - audio_codec="copy", - trim_start=trim_start, - trim_end=trim_end, - ) - - # Wrapper to convert float percent to int - def wrapped_callback(percent: float, details: Dict[str, Any]) -> None: - if progress_callback: - progress_callback(int(percent), details) - - return transcode( - config, - duration=duration, - progress_callback=wrapped_callback if progress_callback else None, + handler = get_handler(job_type) + result = handler.process( + job_id=job_id, + payload=payload, + progress_callback=progress_callback, ) + return result.get("status") == "completed" class LambdaExecutor(Executor): @@ -123,26 +77,18 @@ class LambdaExecutor(Executor): def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: """Start a Step Functions execution for this job.""" import json - payload = { + sfn_payload = { + "job_type": job_type, "job_id": job_id, - "source_key": source_path, - "output_key": output_path, - "preset": preset, - "trim_start": trim_start, - "trim_end": trim_end, - "duration": duration, + **payload, "callback_url": self.callback_url, "api_key": self.callback_api_key, } @@ -150,10 +96,9 @@ class LambdaExecutor(Executor): response = self.sfn.start_execution( stateMachineArn=self.state_machine_arn, name=f"mpr-{job_id}", - input=json.dumps(payload), + input=json.dumps(sfn_payload), ) - # Store execution ARN on the job execution_arn = response["executionArn"] try: from core.db import update_job_fields @@ -179,13 +124,9 @@ class GCPExecutor(Executor): def run( self, + job_type: str, job_id: str, - source_path: str, - output_path: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, + payload: Dict[str, Any], progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, ) -> bool: """Trigger a Cloud Run Job execution for this job.""" @@ -193,14 +134,10 @@ class GCPExecutor(Executor): from google.cloud import run_v2 - payload = { + gcp_payload = { + "job_type": job_type, "job_id": job_id, - "source_key": source_path, - "output_key": output_path, - "preset": preset, - "trim_start": trim_start, - "trim_end": trim_end, - "duration": duration, + **payload, "callback_url": self.callback_url, "api_key": self.callback_api_key, } @@ -216,7 +153,8 @@ class GCPExecutor(Executor): run_v2.RunJobRequest.Overrides.ContainerOverride( env=[ run_v2.EnvVar( - name="MPR_JOB_PAYLOAD", value=json.dumps(payload) + name="MPR_JOB_PAYLOAD", + value=json.dumps(gcp_payload), ) ] ) diff --git a/core/task/gcp_handler.py b/core/jobs/gcp_handler.py similarity index 100% rename from core/task/gcp_handler.py rename to core/jobs/gcp_handler.py diff --git a/core/jobs/handlers/__init__.py b/core/jobs/handlers/__init__.py new file mode 100644 index 0000000..8ac89c6 --- /dev/null +++ b/core/jobs/handlers/__init__.py @@ -0,0 +1,5 @@ +"""Job handlers — type-specific execution logic.""" + +from .base import Handler + +__all__ = ["Handler"] diff --git a/core/jobs/handlers/base.py b/core/jobs/handlers/base.py new file mode 100644 index 0000000..f47328f --- /dev/null +++ b/core/jobs/handlers/base.py @@ -0,0 +1,33 @@ +""" +Base Handler ABC — defines the interface for job-type-specific execution logic. + +A Handler knows HOW to execute a specific kind of job (transcode, chunk, etc.). +The Executor decides WHERE to run it (local, Lambda, GCP). +""" + +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Optional + + +class Handler(ABC): + """Abstract base class for job handlers.""" + + @abstractmethod + def process( + self, + job_id: str, + payload: Dict[str, Any], + progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + """ + Execute job-specific logic. + + Args: + job_id: Unique job identifier + payload: Job-type-specific configuration + progress_callback: Called with (percent, details_dict) + + Returns: + Result dict with at least {"status": "completed"} or raises + """ + pass diff --git a/core/jobs/handlers/chunk.py b/core/jobs/handlers/chunk.py new file mode 100644 index 0000000..7a06a11 --- /dev/null +++ b/core/jobs/handlers/chunk.py @@ -0,0 +1,119 @@ +""" +ChunkHandler — job handler that wraps the chunker Pipeline. + +Downloads source from S3/MinIO, runs FFmpeg chunking pipeline, +uploads mp4 segments + manifest back to S3/MinIO. +""" + +import logging +import os +import shutil +import tempfile +from typing import Any, Callable, Dict, Optional + +from core.chunker import Pipeline +from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file + +from .base import Handler + +logger = logging.getLogger(__name__) + + +class ChunkHandler(Handler): + """ + Handles chunk processing jobs by delegating to the chunker Pipeline. + + Expected payload keys: + source_key: str — S3 key of the source file in BUCKET_IN + chunk_duration: float — seconds per chunk (default: 10.0) + num_workers: int — concurrent workers (default: 4) + max_retries: int — retries per chunk (default: 3) + processor_type: str — "ffmpeg", "checksum", "simulated_decode", "composite" + queue_size: int — max queue depth (default: 10) + """ + + def process( + self, + job_id: str, + payload: Dict[str, Any], + progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + source_key = payload["source_key"] + processor_type = payload.get("processor_type", "ffmpeg") + + logger.info(f"ChunkHandler starting job {job_id}: {source_key}") + + # Download source from S3/MinIO + tmp_source = download_to_temp(BUCKET_IN, source_key) + + # Create temp output directory for chunks + tmp_output_dir = tempfile.mkdtemp(prefix=f"chunks-{job_id}-") + + try: + def event_bridge(event_type: str, data: Dict[str, Any]) -> None: + """Bridge pipeline events to the job progress callback.""" + if progress_callback and event_type == "pipeline_complete": + progress_callback(100, data) + elif progress_callback and event_type == "chunk_done": + total = data.get("total_chunks", 1) + if total > 0: + pct = min(int((data.get("sequence", 0) + 1) / total * 100), 99) + progress_callback(pct, data) + + pipeline = Pipeline( + source=tmp_source, + chunk_duration=payload.get("chunk_duration", 10.0), + num_workers=payload.get("num_workers", 4), + max_retries=payload.get("max_retries", 3), + processor_type=processor_type, + queue_size=payload.get("queue_size", 10), + event_callback=event_bridge, + output_dir=tmp_output_dir if processor_type == "ffmpeg" else None, + ) + + result = pipeline.run() + + # Upload chunks + manifest to S3/MinIO + output_prefix = f"chunks/{job_id}" + uploaded_files = [] + + for chunk_file in result.chunk_files: + filename = os.path.basename(chunk_file) + output_key = f"{output_prefix}/{filename}" + upload_file(chunk_file, BUCKET_OUT, output_key) + uploaded_files.append(output_key) + logger.info(f"Uploaded {output_key}") + + # Upload manifest + manifest_path = os.path.join(tmp_output_dir, "manifest.json") + if os.path.exists(manifest_path): + manifest_key = f"{output_prefix}/manifest.json" + upload_file(manifest_path, BUCKET_OUT, manifest_key) + uploaded_files.append(manifest_key) + logger.info(f"Uploaded {manifest_key}") + + return { + "status": "completed" if result.failed == 0 else "completed_with_errors", + "total_chunks": result.total_chunks, + "processed": result.processed, + "failed": result.failed, + "retries": result.retries, + "elapsed_time": result.elapsed_time, + "throughput_mbps": result.throughput_mbps, + "worker_stats": result.worker_stats, + "errors": result.errors, + "chunks_in_order": result.chunks_in_order, + "output_prefix": output_prefix, + "uploaded_files": uploaded_files, + } + + finally: + # Cleanup temp files + try: + os.unlink(tmp_source) + except OSError: + pass + try: + shutil.rmtree(tmp_output_dir, ignore_errors=True) + except OSError: + pass diff --git a/core/jobs/handlers/transcode.py b/core/jobs/handlers/transcode.py new file mode 100644 index 0000000..6371e2f --- /dev/null +++ b/core/jobs/handlers/transcode.py @@ -0,0 +1,104 @@ +""" +TranscodeHandler — executes transcode/trim jobs using FFmpeg. + +Extracted from the old tasks.py Celery task logic. +""" + +import logging +import os +import tempfile +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +from core.ffmpeg.transcode import TranscodeConfig, transcode +from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file + +from .base import Handler + +logger = logging.getLogger(__name__) + + +class TranscodeHandler(Handler): + """Handle transcode and trim jobs via FFmpeg.""" + + def process( + self, + job_id: str, + payload: Dict[str, Any], + progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + source_key = payload["source_key"] + output_key = payload["output_key"] + preset = payload.get("preset") + trim_start = payload.get("trim_start") + trim_end = payload.get("trim_end") + duration = payload.get("duration") + + logger.info(f"TranscodeHandler: {source_key} -> {output_key}") + + # Download source + tmp_source = download_to_temp(BUCKET_IN, source_key) + + ext = Path(output_key).suffix or ".mp4" + fd, tmp_output = tempfile.mkstemp(suffix=ext) + os.close(fd) + + try: + if preset: + config = TranscodeConfig( + input_path=tmp_source, + output_path=tmp_output, + video_codec=preset.get("video_codec", "libx264"), + video_bitrate=preset.get("video_bitrate"), + video_crf=preset.get("video_crf"), + video_preset=preset.get("video_preset"), + resolution=preset.get("resolution"), + framerate=preset.get("framerate"), + audio_codec=preset.get("audio_codec", "aac"), + audio_bitrate=preset.get("audio_bitrate"), + audio_channels=preset.get("audio_channels"), + audio_samplerate=preset.get("audio_samplerate"), + container=preset.get("container", "mp4"), + extra_args=preset.get("extra_args", []), + trim_start=trim_start, + trim_end=trim_end, + ) + else: + config = TranscodeConfig( + input_path=tmp_source, + output_path=tmp_output, + video_codec="copy", + audio_codec="copy", + trim_start=trim_start, + trim_end=trim_end, + ) + + def wrapped_callback(percent: float, details: Dict[str, Any]) -> None: + if progress_callback: + progress_callback(int(percent), details) + + success = transcode( + config, + duration=duration, + progress_callback=wrapped_callback if progress_callback else None, + ) + + if not success: + raise RuntimeError("Transcode returned False") + + # Upload result + logger.info(f"Uploading {output_key} to {BUCKET_OUT}") + upload_file(tmp_output, BUCKET_OUT, output_key) + + return { + "status": "completed", + "job_id": job_id, + "output_key": output_key, + } + + finally: + for f in [tmp_source, tmp_output]: + try: + os.unlink(f) + except OSError: + pass diff --git a/core/task/lambda_handler.py b/core/jobs/lambda_handler.py similarity index 100% rename from core/task/lambda_handler.py rename to core/jobs/lambda_handler.py diff --git a/core/jobs/registry.py b/core/jobs/registry.py new file mode 100644 index 0000000..6b9b4f7 --- /dev/null +++ b/core/jobs/registry.py @@ -0,0 +1,33 @@ +""" +Handler registry — maps job_type strings to Handler classes. +""" + +from typing import Dict, Type + +from .handlers.base import Handler + +_handlers: Dict[str, Type[Handler]] = {} + + +def register_handler(job_type: str, handler_class: Type[Handler]) -> None: + """Register a handler class for a job type.""" + _handlers[job_type] = handler_class + + +def get_handler(job_type: str) -> Handler: + """Get an instantiated handler for a job type.""" + if job_type not in _handlers: + raise ValueError(f"Unknown job type: {job_type}") + return _handlers[job_type]() + + +def _register_defaults() -> None: + """Register built-in handlers.""" + from .handlers.chunk import ChunkHandler + from .handlers.transcode import TranscodeHandler + + register_handler("transcode", TranscodeHandler) + register_handler("chunk", ChunkHandler) + + +_register_defaults() diff --git a/core/jobs/task.py b/core/jobs/task.py new file mode 100644 index 0000000..4c0a60d --- /dev/null +++ b/core/jobs/task.py @@ -0,0 +1,64 @@ +""" +Celery task for job processing. + +Generic dispatcher — routes to the appropriate handler based on job_type. +""" + +import logging +from typing import Any, Dict + +from celery import shared_task + +from core.rpc.server import update_job_progress + +logger = logging.getLogger(__name__) + + +@shared_task(bind=True, max_retries=3, default_retry_delay=60) +def run_job( + self, + job_type: str, + job_id: str, + payload: Dict[str, Any], +) -> Dict[str, Any]: + """ + Generic Celery task — dispatches to the registered handler for job_type. + """ + logger.info(f"Starting {job_type} job {job_id}") + + update_job_progress(job_id, progress=0, status="processing") + + def progress_callback(percent: int, details: Dict[str, Any]) -> None: + update_job_progress( + job_id, + progress=percent, + current_time=details.get("time", 0.0), + status="processing", + ) + + try: + from .registry import get_handler + + handler = get_handler(job_type) + result = handler.process( + job_id=job_id, + payload=payload, + progress_callback=progress_callback, + ) + + logger.info(f"Job {job_id} completed successfully") + update_job_progress(job_id, progress=100, status="completed") + return result + + except Exception as e: + logger.exception(f"Job {job_id} failed: {e}") + update_job_progress(job_id, progress=0, status="failed", error=str(e)) + + if self.request.retries < self.max_retries: + raise self.retry(exc=e) + + return { + "status": "failed", + "job_id": job_id, + "error": str(e), + } diff --git a/core/rpc/server.py b/core/rpc/server.py index f4ae778..d07ce61 100644 --- a/core/rpc/server.py +++ b/core/rpc/server.py @@ -59,17 +59,24 @@ class WorkerServicer(worker_pb2_grpc.WorkerServiceServicer): # Dispatch to Celery if available if self.celery_app: - from core.task.tasks import run_transcode_job + from core.jobs.task import run_job - task = run_transcode_job.delay( - job_id=job_id, - source_path=request.source_path, - output_path=request.output_path, - preset=preset, - trim_start=request.trim_start + payload = { + "source_key": request.source_path, + "output_key": request.output_path, + "preset": preset, + "trim_start": request.trim_start if request.HasField("trim_start") else None, - trim_end=request.trim_end if request.HasField("trim_end") else None, + "trim_end": request.trim_end + if request.HasField("trim_end") + else None, + } + + task = run_job.delay( + job_type="transcode", + job_id=job_id, + payload=payload, ) _active_jobs[job_id]["celery_task_id"] = task.id @@ -197,11 +204,14 @@ def update_job_progress( speed: float = 0.0, status: str = "processing", error: str = None, + **extra, ) -> None: """ Update job progress (called from worker tasks). Updates both the in-memory gRPC state and the Django database. + Extra kwargs are stored for chunker-specific fields (total_chunks, + processed_chunks, failed_chunks, throughput_mbps, etc.). """ if job_id in _active_jobs: _active_jobs[job_id].update( @@ -212,6 +222,7 @@ def update_job_progress( "speed": speed, "status": status, "error": error, + **extra, } ) diff --git a/core/schema/models/__init__.py b/core/schema/models/__init__.py index 188440f..b0b0dcc 100644 --- a/core/schema/models/__init__.py +++ b/core/schema/models/__init__.py @@ -23,12 +23,12 @@ from .grpc import ( ProgressUpdate, WorkerStatus, ) -from .jobs import JobStatus, TranscodeJob +from .jobs import ChunkJob, ChunkJobStatus, JobStatus, TranscodeJob from .media import AssetStatus, MediaAsset from .presets import BUILTIN_PRESETS, TranscodePreset # Core domain models - generates Django, Pydantic, TypeScript -DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob] +DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob, ChunkJob] # API request/response models - generates TypeScript only (no Django) # WorkerStatus from grpc.py is reused here @@ -42,7 +42,7 @@ API_MODELS = [ ] # Status enums - included in generated code -ENUMS = [AssetStatus, JobStatus] +ENUMS = [AssetStatus, JobStatus, ChunkJobStatus] # gRPC messages - generates Proto GRPC_MESSAGES = [ @@ -61,6 +61,7 @@ __all__ = [ "MediaAsset", "TranscodePreset", "TranscodeJob", + "ChunkJob", # API Models "CreateJobRequest", "UpdateAssetRequest", @@ -70,6 +71,7 @@ __all__ = [ # Enums "AssetStatus", "JobStatus", + "ChunkJobStatus", # gRPC "GRPC_SERVICE", "JobRequest", diff --git a/core/schema/models/jobs.py b/core/schema/models/jobs.py index 8a7f6f0..0957034 100644 --- a/core/schema/models/jobs.py +++ b/core/schema/models/jobs.py @@ -1,13 +1,14 @@ """ -TranscodeJob Schema Definition +Job Schema Definitions -Source of truth for job data model. +Source of truth for job data models. +TranscodeJob and ChunkJob share common lifecycle fields by convention. """ from dataclasses import dataclass, field from datetime import datetime from enum import Enum -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from uuid import UUID @@ -77,3 +78,56 @@ class TranscodeJob: return self.preset_id is None and ( self.trim_start is not None or self.trim_end is not None ) + + +class ChunkJobStatus(str, Enum): + """Status of a chunk pipeline job.""" + + PENDING = "pending" + CHUNKING = "chunking" + PROCESSING = "processing" + COLLECTING = "collecting" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass +class ChunkJob: + """ + A chunk pipeline job — splits a media file into chunks and processes them + through a concurrent worker pool. + """ + + id: UUID + + # Input + source_asset_id: UUID + + # Configuration + chunk_duration: float = 10.0 # seconds + num_workers: int = 4 + max_retries: int = 3 + processor_type: str = "ffmpeg" # "ffmpeg", "checksum", "simulated_decode", "composite" + + # Status & Progress + status: ChunkJobStatus = ChunkJobStatus.PENDING + progress: float = 0.0 # 0.0 to 100.0 + total_chunks: int = 0 + processed_chunks: int = 0 + failed_chunks: int = 0 + retry_count: int = 0 + error_message: Optional[str] = None + + # Result stats + throughput_mbps: Optional[float] = None + elapsed_seconds: Optional[float] = None + + # Worker tracking + celery_task_id: Optional[str] = None + priority: int = 0 # Lower = higher priority + + # Timestamps + created_at: Optional[datetime] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None diff --git a/core/task/__init__.py b/core/task/__init__.py deleted file mode 100644 index fea2bad..0000000 --- a/core/task/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -MPR Worker Module - -Provides executor abstraction and Celery tasks for job processing. -""" - -from .executor import Executor, LocalExecutor, get_executor -from .tasks import run_transcode_job - -__all__ = [ - "Executor", - "LocalExecutor", - "get_executor", - "run_transcode_job", -] diff --git a/core/task/tasks.py b/core/task/tasks.py deleted file mode 100644 index 5e0713a..0000000 --- a/core/task/tasks.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -Celery tasks for job processing. -""" - -import logging -import os -from typing import Any, Dict, Optional - -from celery import shared_task - -from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file -from core.rpc.server import update_job_progress -from core.task.executor import get_executor - -logger = logging.getLogger(__name__) - - -@shared_task(bind=True, queue="transcode", max_retries=3, default_retry_delay=60) -def run_transcode_job( - self, - job_id: str, - source_key: str, - output_key: str, - preset: Optional[Dict[str, Any]] = None, - trim_start: Optional[float] = None, - trim_end: Optional[float] = None, - duration: Optional[float] = None, -) -> Dict[str, Any]: - """ - Celery task to run a transcode/trim job. - - Downloads source from S3, runs FFmpeg, uploads result to S3. - """ - logger.info(f"Starting job {job_id}: {source_key} -> {output_key}") - - update_job_progress(job_id, progress=0, status="processing") - - # Download source from S3 to temp file - logger.info(f"Downloading {source_key} from {BUCKET_IN}") - tmp_source = download_to_temp(BUCKET_IN, source_key) - - # Create temp output path with same extension - import tempfile - from pathlib import Path - - ext = Path(output_key).suffix or ".mp4" - fd, tmp_output = tempfile.mkstemp(suffix=ext) - os.close(fd) - - def progress_callback(percent: int, details: Dict[str, Any]) -> None: - update_job_progress( - job_id, - progress=percent, - current_time=details.get("time", 0.0), - status="processing", - ) - - try: - executor = get_executor() - success = executor.run( - job_id=job_id, - source_path=tmp_source, - output_path=tmp_output, - preset=preset, - trim_start=trim_start, - trim_end=trim_end, - duration=duration, - progress_callback=progress_callback, - ) - - if success: - # Upload result to S3 - logger.info(f"Uploading {output_key} to {BUCKET_OUT}") - upload_file(tmp_output, BUCKET_OUT, output_key) - - logger.info(f"Job {job_id} completed successfully") - update_job_progress(job_id, progress=100, status="completed") - return { - "status": "completed", - "job_id": job_id, - "output_key": output_key, - } - else: - raise RuntimeError("Executor returned False") - - except Exception as e: - logger.exception(f"Job {job_id} failed: {e}") - update_job_progress(job_id, progress=0, status="failed", error=str(e)) - - if self.request.retries < self.max_retries: - raise self.retry(exc=e) - - return { - "status": "failed", - "job_id": job_id, - "error": str(e), - } - - finally: - # Clean up temp files - for f in [tmp_source, tmp_output]: - try: - os.unlink(f) - except OSError: - pass diff --git a/ctrl/Dockerfile b/ctrl/Dockerfile index 80e2152..247b054 100644 --- a/ctrl/Dockerfile +++ b/ctrl/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY . . +# No COPY . . — code is volume-mounted in dev (..:/app) +# This image only provides the Python runtime + dependencies CMD ["python", "admin/manage.py", "runserver", "0.0.0.0:8000"] diff --git a/ctrl/Dockerfile.worker b/ctrl/Dockerfile.worker index bbb7a49..e69ff5b 100644 --- a/ctrl/Dockerfile.worker +++ b/ctrl/Dockerfile.worker @@ -9,6 +9,7 @@ WORKDIR /app COPY requirements.txt requirements-worker.txt ./ RUN pip install --no-cache-dir -r requirements-worker.txt -COPY . . +# No COPY . . — code is volume-mounted in dev (..:/app) +# This image only provides Python runtime + FFmpeg + dependencies CMD ["celery", "-A", "admin.mpr", "worker", "--loglevel=info"] diff --git a/ctrl/docker-compose.yml b/ctrl/docker-compose.yml index 2c114c7..bfbf700 100644 --- a/ctrl/docker-compose.yml +++ b/ctrl/docker-compose.yml @@ -17,6 +17,20 @@ x-healthcheck-defaults: &healthcheck-defaults timeout: 5s retries: 5 +x-python-service: &python-service + build: + context: .. + dockerfile: ctrl/Dockerfile + volumes: + - ..:/app + environment: + <<: *common-env + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + services: # ============================================================================= # Infrastructure @@ -92,47 +106,25 @@ services: # ============================================================================= django: - build: - context: .. - dockerfile: ctrl/Dockerfile + <<: *python-service command: > bash -c "python admin/manage.py migrate && python admin/manage.py loadbuiltins || true && python admin/manage.py runserver 0.0.0.0:8701" ports: - "8701:8701" - environment: - <<: *common-env - volumes: - - ..:/app - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy fastapi: - build: - context: .. - dockerfile: ctrl/Dockerfile + <<: *python-service command: uvicorn core.api.main:app --host 0.0.0.0 --port 8702 --reload ports: - "8702:8702" environment: <<: *common-env DJANGO_ALLOW_ASYNC_UNSAFE: "true" - volumes: - - ..:/app - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy grpc: - build: - context: .. - dockerfile: ctrl/Dockerfile + <<: *python-service command: python -m core.rpc.server ports: - "50052:50051" @@ -140,13 +132,6 @@ services: <<: *common-env GRPC_PORT: 50051 GRPC_MAX_WORKERS: 10 - volumes: - - ..:/app - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy celery: build: diff --git a/ctrl/lambda/Dockerfile b/ctrl/lambda/Dockerfile index 52c0b82..2869686 100644 --- a/ctrl/lambda/Dockerfile +++ b/ctrl/lambda/Dockerfile @@ -14,8 +14,8 @@ COPY ctrl/lambda/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # Copy application code -COPY core/task/lambda_handler.py ${LAMBDA_TASK_ROOT}/core/task/lambda_handler.py -COPY core/task/__init__.py ${LAMBDA_TASK_ROOT}/core/task/__init__.py +COPY core/jobs/lambda_handler.py ${LAMBDA_TASK_ROOT}/core/jobs/lambda_handler.py +COPY core/jobs/__init__.py ${LAMBDA_TASK_ROOT}/core/jobs/__init__.py COPY core/ ${LAMBDA_TASK_ROOT}/core/ -CMD ["core.task.lambda_handler.handler"] +CMD ["core.jobs.lambda_handler.handler"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/chunker/__init__.py b/tests/chunker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/chunker/conftest.py b/tests/chunker/conftest.py new file mode 100644 index 0000000..1ddafe6 --- /dev/null +++ b/tests/chunker/conftest.py @@ -0,0 +1,76 @@ +""" +Shared fixtures for chunker tests. + +Demonstrates: TDD and unit testing best practices (Interview Topic 8) — fixtures, temp files. +""" + +import os +import tempfile + +import pytest + +from core.chunker.models import Chunk, ChunkResult + + +@pytest.fixture +def temp_file(): + """Create a temporary file with known content, cleaned up after test.""" + files = [] + + def _create(content: bytes = b"x" * 4096): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") + f.write(content) + f.close() + files.append(f.name) + return f.name + + yield _create + + for path in files: + if os.path.exists(path): + os.unlink(path) + + +@pytest.fixture +def sample_chunk(temp_file): + """Create a sample time-based Chunk with valid time range.""" + path = temp_file(b"x" * 1024) + return Chunk( + sequence=0, + start_time=0.0, + end_time=10.0, + source_path=path, + duration=10.0, + ) + + +@pytest.fixture +def make_chunk(temp_file): + """Factory fixture for creating time-based chunks with specific sequence numbers.""" + path = temp_file(b"x" * 1024) + + def _make(sequence: int, duration: float = 10.0) -> Chunk: + start = sequence * duration + return Chunk( + sequence=sequence, + start_time=start, + end_time=start + duration, + source_path=path, + duration=duration, + ) + + return _make + + +@pytest.fixture +def make_result(): + """Factory fixture for creating ChunkResults.""" + + def _make(sequence: int, success: bool = True, processing_time: float = 0.01) -> ChunkResult: + return ChunkResult( + sequence=sequence, + success=success, + processing_time=processing_time, + ) + + return _make diff --git a/tests/chunker/test_chunker.py b/tests/chunker/test_chunker.py new file mode 100644 index 0000000..7f132fa --- /dev/null +++ b/tests/chunker/test_chunker.py @@ -0,0 +1,149 @@ +""" +Tests for Chunker — time-based segmentation, chunk counts, sequence numbers, generator behavior. + +Demonstrates: TDD (Interview Topic 8) — parametrized tests, edge cases, mocking. +""" + +from unittest.mock import patch, MagicMock + +import pytest + +from core.chunker import Chunker +from core.chunker.exceptions import ChunkReadError + + +def mock_probe(duration): + """Create a mock probe_file that returns the given duration.""" + result = MagicMock() + result.duration = duration + return result + + +class TestChunker: + @patch("core.chunker.chunker.probe_file") + def test_basic_chunking(self, mock_pf, temp_file): + """File splits into expected number of time-based chunks.""" + path = temp_file(b"x" * 1000) + mock_pf.return_value = mock_probe(30.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + + assert len(chunks) == 3 + assert chunks[0].start_time == 0.0 + assert chunks[0].end_time == 10.0 + assert chunks[0].duration == 10.0 + assert chunks[1].start_time == 10.0 + assert chunks[2].start_time == 20.0 + + @patch("core.chunker.chunker.probe_file") + def test_sequence_numbers(self, mock_pf, temp_file): + """Chunks have sequential sequence numbers starting at 0.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(40.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + sequences = [c.sequence for c in chunks] + + assert sequences == [0, 1, 2, 3] + + @patch("core.chunker.chunker.probe_file") + def test_time_ranges(self, mock_pf, temp_file): + """Each chunk has correct start_time and end_time.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(25.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + + assert chunks[0].start_time == 0.0 + assert chunks[0].end_time == 10.0 + assert chunks[1].start_time == 10.0 + assert chunks[1].end_time == 20.0 + assert chunks[2].start_time == 20.0 + assert chunks[2].end_time == 25.0 # last chunk shorter + assert chunks[2].duration == 5.0 + + @patch("core.chunker.chunker.probe_file") + def test_expected_chunks_property(self, mock_pf, temp_file): + """expected_chunks calculates correctly before iteration.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(25.0) + + chunker = Chunker(path, chunk_duration=10.0) + assert chunker.expected_chunks == 3 # ceil(25/10) + + @patch("core.chunker.chunker.probe_file") + def test_source_path_on_chunks(self, mock_pf, temp_file): + """Each chunk carries the source file path.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(10.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + + assert all(c.source_path == path for c in chunks) + + def test_file_not_found(self): + """Non-existent file raises ChunkReadError.""" + with pytest.raises(ChunkReadError, match="File not found"): + Chunker("/nonexistent/file.mp4") + + @patch("core.chunker.chunker.probe_file") + def test_invalid_chunk_duration(self, mock_pf, temp_file): + """Zero or negative chunk_duration raises ValueError.""" + path = temp_file(b"x" * 100) + + with pytest.raises(ValueError, match="chunk_duration must be positive"): + Chunker(path, chunk_duration=0) + + with pytest.raises(ValueError, match="chunk_duration must be positive"): + Chunker(path, chunk_duration=-1) + + @patch("core.chunker.chunker.probe_file") + def test_generator_laziness(self, mock_pf, temp_file): + """Chunks are yielded lazily, not pre-loaded.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(30.0) + + chunker = Chunker(path, chunk_duration=10.0) + gen = chunker.chunks() + first = next(gen) + assert first.sequence == 0 + # Generator is not exhausted — remaining chunks still pending + + @pytest.mark.parametrize("duration,chunk_dur,expected", [ + (10.0, 10.0, 1), + (10.1, 10.0, 2), + (1.0, 1.0, 1), + (100.0, 1.0, 100), + (5.0, 100.0, 1), + ]) + @patch("core.chunker.chunker.probe_file") + def test_expected_chunks_parametrized(self, mock_pf, temp_file, duration, chunk_dur, expected): + """Parametrized: various duration/chunk_duration combos.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(duration) + chunker = Chunker(path, chunk_duration=chunk_dur) + assert chunker.expected_chunks == expected + + @patch("core.chunker.chunker.probe_file") + def test_exact_multiple(self, mock_pf, temp_file): + """Duration exactly divisible by chunk_duration.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(30.0) + + chunker = Chunker(path, chunk_duration=10.0) + chunks = list(chunker.chunks()) + assert len(chunks) == 3 + assert all(c.duration == 10.0 for c in chunks) + + @patch("core.chunker.chunker.probe_file") + def test_probe_failure(self, mock_pf, temp_file): + """Probe failure raises ChunkReadError.""" + path = temp_file(b"x" * 100) + mock_pf.side_effect = Exception("ffprobe failed") + + with pytest.raises(ChunkReadError, match="Failed to probe"): + Chunker(path, chunk_duration=10.0) diff --git a/tests/chunker/test_collector.py b/tests/chunker/test_collector.py new file mode 100644 index 0000000..8dc1e64 --- /dev/null +++ b/tests/chunker/test_collector.py @@ -0,0 +1,103 @@ +""" +Tests for ResultCollector — ordered reassembly, out-of-order buffering, duplicates. + +Demonstrates: TDD (Interview Topic 8) — testing algorithms (heapq reassembly). +""" + +import pytest + +from core.chunker.collector import ResultCollector +from core.chunker.exceptions import ReassemblyError + + +class TestResultCollector: + def test_in_order_emission(self, make_result): + """Results arriving in order are emitted immediately.""" + collector = ResultCollector(total_chunks=3) + + emitted = collector.add(make_result(0)) + assert len(emitted) == 1 + assert emitted[0].sequence == 0 + + emitted = collector.add(make_result(1)) + assert len(emitted) == 1 + + emitted = collector.add(make_result(2)) + assert len(emitted) == 1 + + assert collector.is_complete + + def test_out_of_order_buffering(self, make_result): + """Out-of-order results are buffered until gaps fill.""" + collector = ResultCollector(total_chunks=3) + + # Arrive: 2, 0, 1 + emitted = collector.add(make_result(2)) + assert len(emitted) == 0 + assert collector.buffered_count == 1 + + emitted = collector.add(make_result(0)) + assert len(emitted) == 1 # Only 0 emitted, 1 still missing + + emitted = collector.add(make_result(1)) + assert len(emitted) == 2 # 1 and 2 now emittable + assert collector.is_complete + + def test_reverse_order(self, make_result): + """All results arrive in reverse — only last add emits everything.""" + collector = ResultCollector(total_chunks=4) + + for seq in [3, 2, 1]: + emitted = collector.add(make_result(seq)) + assert len(emitted) == 0 + + emitted = collector.add(make_result(0)) + assert len(emitted) == 4 + assert collector.is_complete + + def test_duplicate_raises(self, make_result): + """Duplicate sequence number raises ReassemblyError.""" + collector = ResultCollector(total_chunks=3) + collector.add(make_result(0)) + + with pytest.raises(ReassemblyError, match="Duplicate"): + collector.add(make_result(0)) + + def test_emitted_count(self, make_result): + """emitted_count tracks correctly.""" + collector = ResultCollector(total_chunks=3) + assert collector.emitted_count == 0 + + collector.add(make_result(0)) + assert collector.emitted_count == 1 + + collector.add(make_result(2)) # buffered + assert collector.emitted_count == 1 + + collector.add(make_result(1)) # releases 1 and 2 + assert collector.emitted_count == 3 + + def test_get_ordered_results(self, make_result): + """get_ordered_results returns all emitted results in order.""" + collector = ResultCollector(total_chunks=3) + collector.add(make_result(2)) + collector.add(make_result(0)) + collector.add(make_result(1)) + + ordered = collector.get_ordered_results() + assert [r.sequence for r in ordered] == [0, 1, 2] + + def test_avg_processing_time(self, make_result): + """Average processing time from sliding window.""" + collector = ResultCollector(total_chunks=2) + collector.add(make_result(0, processing_time=0.1)) + collector.add(make_result(1, processing_time=0.3)) + + assert abs(collector.avg_processing_time - 0.2) < 0.001 + + def test_not_complete_when_partial(self, make_result): + """is_complete is False until all chunks emitted.""" + collector = ResultCollector(total_chunks=3) + collector.add(make_result(0)) + collector.add(make_result(1)) + assert not collector.is_complete diff --git a/tests/chunker/test_exceptions.py b/tests/chunker/test_exceptions.py new file mode 100644 index 0000000..91ff59e --- /dev/null +++ b/tests/chunker/test_exceptions.py @@ -0,0 +1,69 @@ +""" +Tests for exception hierarchy — catch patterns, attributes. + +Demonstrates: TDD (Interview Topic 8) — testing exception design. +""" + +import pytest + +from core.chunker.exceptions import ( + ChunkChecksumError, + ChunkError, + ChunkReadError, + PipelineError, + ProcessingError, + ProcessorFailureError, + ProcessorTimeoutError, + ReassemblyError, +) + + +class TestExceptionHierarchy: + """Verify the exception class hierarchy and catch patterns.""" + + def test_pipeline_error_is_base(self): + """All chunker exceptions inherit from PipelineError.""" + assert issubclass(ChunkError, PipelineError) + assert issubclass(ProcessingError, PipelineError) + assert issubclass(ReassemblyError, PipelineError) + + def test_chunk_error_subtypes(self): + """ChunkReadError and ChunkChecksumError are ChunkErrors.""" + assert issubclass(ChunkReadError, ChunkError) + assert issubclass(ChunkChecksumError, ChunkError) + + def test_processing_error_subtypes(self): + """ProcessorTimeoutError and ProcessorFailureError are ProcessingErrors.""" + assert issubclass(ProcessorTimeoutError, ProcessingError) + assert issubclass(ProcessorFailureError, ProcessingError) + + def test_catch_pipeline_error_catches_all(self): + """Catching PipelineError catches any subtype.""" + with pytest.raises(PipelineError): + raise ChunkReadError("test") + + with pytest.raises(PipelineError): + raise ReassemblyError("test") + + def test_checksum_error_attributes(self): + """ChunkChecksumError carries sequence, expected, actual.""" + err = ChunkChecksumError(sequence=5, expected="aaa", actual="bbb") + assert err.sequence == 5 + assert err.expected == "aaa" + assert err.actual == "bbb" + assert "5" in str(err) + + def test_timeout_error_attributes(self): + """ProcessorTimeoutError carries sequence and timeout.""" + err = ProcessorTimeoutError(sequence=3, timeout=30.0) + assert err.sequence == 3 + assert err.timeout == 30.0 + + def test_failure_error_attributes(self): + """ProcessorFailureError carries sequence, retries, original error.""" + original = RuntimeError("boom") + err = ProcessorFailureError(sequence=1, retries=3, original_error=original) + assert err.sequence == 1 + assert err.retries == 3 + assert err.original_error is original + assert "boom" in str(err) diff --git a/tests/chunker/test_pipeline.py b/tests/chunker/test_pipeline.py new file mode 100644 index 0000000..e12e2d9 --- /dev/null +++ b/tests/chunker/test_pipeline.py @@ -0,0 +1,144 @@ +""" +Tests for Pipeline — end-to-end orchestration, stats, error handling. + +Demonstrates: TDD (Interview Topic 8) — integration testing with mocked FFmpeg probe. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from core.chunker import Pipeline +from core.chunker.exceptions import PipelineError + + +def mock_probe(duration): + """Create a mock ProbeResult with the given duration.""" + result = MagicMock() + result.duration = duration + return result + + +class TestPipeline: + @patch("core.chunker.chunker.probe_file") + def test_end_to_end(self, mock_pf, temp_file): + """Full pipeline processes a file successfully.""" + path = temp_file(b"x" * 4096) + mock_pf.return_value = mock_probe(40.0) + + result = Pipeline( + source=path, + chunk_duration=10.0, + num_workers=2, + processor_type="checksum", + ).run() + + assert result.total_chunks == 4 + assert result.processed == 4 + assert result.failed == 0 + assert result.elapsed_time > 0 + assert result.chunks_in_order is True + + @patch("core.chunker.chunker.probe_file") + def test_throughput_calculated(self, mock_pf, temp_file): + """Pipeline calculates throughput.""" + path = temp_file(b"x" * 10000) + mock_pf.return_value = mock_probe(30.0) + + result = Pipeline(source=path, chunk_duration=10.0, num_workers=2).run() + + assert result.throughput_mbps > 0 + + @patch("core.chunker.chunker.probe_file") + def test_worker_stats(self, mock_pf, temp_file): + """Pipeline reports per-worker stats.""" + path = temp_file(b"x" * 4000) + mock_pf.return_value = mock_probe(40.0) + + result = Pipeline( + source=path, chunk_duration=10.0, num_workers=2 + ).run() + + assert len(result.worker_stats) == 2 + for worker_id, stats in result.worker_stats.items(): + assert "processed" in stats + assert "errors" in stats + + def test_nonexistent_file(self): + """Non-existent file raises PipelineError.""" + with pytest.raises(PipelineError): + Pipeline(source="/nonexistent/file.mp4").run() + + @patch("core.chunker.chunker.probe_file") + def test_event_callback(self, mock_pf, temp_file): + """Pipeline emits events through callback.""" + path = temp_file(b"x" * 2048) + mock_pf.return_value = mock_probe(20.0) + events = [] + + def capture(event_type, data): + events.append(event_type) + + Pipeline( + source=path, + chunk_duration=10.0, + num_workers=1, + event_callback=capture, + ).run() + + assert "pipeline_start" in events + assert "pipeline_complete" in events + assert "chunk_queued" in events + + @patch("core.chunker.chunker.probe_file") + def test_simulated_decode_processor(self, mock_pf, temp_file): + """Pipeline works with simulated_decode processor.""" + path = temp_file(b"x" * 2048) + mock_pf.return_value = mock_probe(20.0) + + result = Pipeline( + source=path, + chunk_duration=10.0, + num_workers=2, + processor_type="simulated_decode", + ).run() + + assert result.total_chunks == 2 + assert result.failed == 0 + + @patch("core.chunker.chunker.probe_file") + def test_single_chunk_file(self, mock_pf, temp_file): + """Duration shorter than chunk_duration produces one chunk.""" + path = temp_file(b"x" * 100) + mock_pf.return_value = mock_probe(5.0) + + result = Pipeline(source=path, chunk_duration=10.0).run() + + assert result.total_chunks == 1 + assert result.processed == 1 + + @patch("core.chunker.chunker.probe_file") + def test_retries_tracked(self, mock_pf, temp_file): + """Pipeline result tracks total retries.""" + path = temp_file(b"x" * 2048) + mock_pf.return_value = mock_probe(20.0) + + result = Pipeline(source=path, chunk_duration=10.0).run() + + assert result.retries >= 0 # Might be 0 if no failures + + @patch("core.chunker.chunker.probe_file") + def test_output_dir_and_chunk_files(self, mock_pf, temp_file): + """Pipeline tracks output_dir and chunk_files when set.""" + path = temp_file(b"x" * 1024) + mock_pf.return_value = mock_probe(10.0) + + result = Pipeline( + source=path, + chunk_duration=10.0, + processor_type="checksum", + ).run() + + # No output_dir set, so chunk_files should be empty + assert result.output_dir is None + assert result.chunk_files == [] diff --git a/tests/chunker/test_processor.py b/tests/chunker/test_processor.py new file mode 100644 index 0000000..68980f2 --- /dev/null +++ b/tests/chunker/test_processor.py @@ -0,0 +1,98 @@ +""" +Tests for Processor implementations — ChecksumProcessor, SimulatedDecodeProcessor, CompositeProcessor. + +Demonstrates: TDD (Interview Topic 8) — ABC contract, parametrized tests. +""" + +import pytest + +from core.chunker.exceptions import ChunkChecksumError +from core.chunker.models import Chunk +from core.chunker.processor import ( + ChecksumProcessor, + CompositeProcessor, + Processor, + SimulatedDecodeProcessor, +) + + +class TestChecksumProcessor: + def test_valid_time_range(self, sample_chunk): + """Valid time range passes.""" + proc = ChecksumProcessor() + result = proc.process(sample_chunk) + assert result.success is True + assert result.checksum_valid is True + assert result.processing_time > 0 + + def test_invalid_time_range(self): + """Invalid time range raises ChunkChecksumError.""" + chunk = Chunk( + sequence=0, + start_time=10.0, + end_time=10.0, # zero duration + source_path="/fake.mp4", + duration=0.0, + ) + proc = ChecksumProcessor() + with pytest.raises(ChunkChecksumError) as exc_info: + proc.process(chunk) + assert exc_info.value.sequence == 0 + + def test_sequence_preserved(self, make_chunk): + """Result carries the chunk's sequence number.""" + chunk = make_chunk(42) + proc = ChecksumProcessor() + result = proc.process(chunk) + assert result.sequence == 42 + + +class TestSimulatedDecodeProcessor: + def test_processes_successfully(self, sample_chunk): + """Simulated decode always succeeds.""" + proc = SimulatedDecodeProcessor(ms_per_second=1.0) + result = proc.process(sample_chunk) + assert result.success is True + assert result.processing_time > 0 + + def test_time_proportional_to_duration(self): + """Longer chunks take longer.""" + short = Chunk(0, 0.0, 1.0, "/fake.mp4", 1.0) + long = Chunk(1, 0.0, 10.0, "/fake.mp4", 10.0) + + proc = SimulatedDecodeProcessor(ms_per_second=50.0) + r_short = proc.process(short) + r_long = proc.process(long) + + assert r_long.processing_time > r_short.processing_time + + +class TestCompositeProcessor: + def test_chains_processors(self, sample_chunk): + """Composite runs all processors in sequence.""" + proc = CompositeProcessor([ + ChecksumProcessor(), + SimulatedDecodeProcessor(ms_per_second=1.0), + ]) + result = proc.process(sample_chunk) + assert result.success is True + + def test_stops_on_failure(self): + """If first processor raises, composite propagates the error.""" + bad_chunk = Chunk(0, 10.0, 10.0, "/fake.mp4", 0.0) # invalid range + proc = CompositeProcessor([ + ChecksumProcessor(), + SimulatedDecodeProcessor(ms_per_second=1.0), + ]) + with pytest.raises(ChunkChecksumError): + proc.process(bad_chunk) + + def test_requires_at_least_one(self): + """Empty processor list raises ValueError.""" + with pytest.raises(ValueError, match="at least one"): + CompositeProcessor([]) + + def test_is_processor(self): + """CompositeProcessor is a Processor.""" + proc = CompositeProcessor([ChecksumProcessor()]) + assert isinstance(proc, Processor) diff --git a/tests/chunker/test_queue.py b/tests/chunker/test_queue.py new file mode 100644 index 0000000..7ebee6b --- /dev/null +++ b/tests/chunker/test_queue.py @@ -0,0 +1,115 @@ +""" +Tests for ChunkQueue — backpressure, sentinel shutdown, timeout behavior. + +Demonstrates: TDD (Interview Topic 8) — concurrency testing. +""" + +import queue +import threading + +import pytest + +from core.chunker.queue import ChunkQueue + + +class TestChunkQueue: + def test_put_and_get(self, make_chunk): + """Basic put/get cycle.""" + q = ChunkQueue(maxsize=5) + chunk = make_chunk(0) + q.put(chunk) + result = q.get(timeout=1.0) + assert result.sequence == 0 + + def test_fifo_order(self, make_chunk): + """Items come out in FIFO order.""" + q = ChunkQueue(maxsize=5) + for i in range(3): + q.put(make_chunk(i)) + + for i in range(3): + assert q.get(timeout=1.0).sequence == i + + def test_close_returns_none(self, make_chunk): + """After close(), get() returns None (sentinel).""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + result = q.get(timeout=1.0) + assert result.sequence == 0 + + # Next get should hit sentinel + result = q.get(timeout=1.0) + assert result is None + + def test_close_propagates_to_multiple_consumers(self, make_chunk): + """Sentinel propagates: multiple consumers all get None.""" + q = ChunkQueue(maxsize=5) + q.close() + + # Multiple consumers should all see None + assert q.get(timeout=1.0) is None + assert q.get(timeout=1.0) is None + + def test_is_closed(self): + """is_closed reflects state.""" + q = ChunkQueue() + assert not q.is_closed + q.close() + assert q.is_closed + + def test_qsize(self, make_chunk): + """qsize tracks approximate queue depth.""" + q = ChunkQueue(maxsize=10) + assert q.qsize() == 0 + + q.put(make_chunk(0)) + q.put(make_chunk(1)) + assert q.qsize() == 2 + + q.get(timeout=1.0) + assert q.qsize() == 1 + + def test_backpressure_blocks(self, make_chunk): + """Put blocks when queue is full (backpressure).""" + q = ChunkQueue(maxsize=2) + q.put(make_chunk(0)) + q.put(make_chunk(1)) + + # Queue is full — put with short timeout should raise + with pytest.raises(queue.Full): + q.put(make_chunk(2), timeout=0.05) + + def test_get_timeout(self): + """Get on empty queue with timeout raises Empty.""" + q = ChunkQueue(maxsize=5) + + with pytest.raises(queue.Empty): + q.get(timeout=0.05) + + def test_concurrent_put_get(self, make_chunk): + """Producer/consumer threads work correctly.""" + q = ChunkQueue(maxsize=3) + results = [] + + def producer(): + for i in range(10): + q.put(make_chunk(i)) + q.close() + + def consumer(): + while True: + item = q.get(timeout=2.0) + if item is None: + break + results.append(item.sequence) + + t1 = threading.Thread(target=producer) + t2 = threading.Thread(target=consumer) + t1.start() + t2.start() + t1.join(timeout=5.0) + t2.join(timeout=5.0) + + assert sorted(results) == list(range(10)) diff --git a/tests/chunker/test_worker.py b/tests/chunker/test_worker.py new file mode 100644 index 0000000..12af394 --- /dev/null +++ b/tests/chunker/test_worker.py @@ -0,0 +1,127 @@ +""" +Tests for Worker — processing, retry with backoff, error handling. + +Demonstrates: TDD (Interview Topic 8) — mocking processors, testing retry logic. +""" + +from unittest.mock import MagicMock + +import pytest + +from core.chunker.models import Chunk, ChunkResult +from core.chunker.processor import Processor +from core.chunker.queue import ChunkQueue +from core.chunker.worker import Worker + + +class FailNTimesProcessor(Processor): + """Test processor that fails N times then succeeds.""" + + def __init__(self, fail_count: int): + self.fail_count = fail_count + self.call_count = 0 + + def process(self, chunk: Chunk) -> ChunkResult: + self.call_count += 1 + if self.call_count <= self.fail_count: + raise RuntimeError(f"Simulated failure #{self.call_count}") + return ChunkResult( + sequence=chunk.sequence, + success=True, + processing_time=0.001, + ) + + +class AlwaysFailProcessor(Processor): + """Test processor that always fails.""" + + def process(self, chunk: Chunk) -> ChunkResult: + raise RuntimeError("Always fails") + + +class TestWorker: + def test_processes_chunks(self, make_chunk): + """Worker processes all chunks from queue.""" + q = ChunkQueue(maxsize=5) + for i in range(3): + q.put(make_chunk(i)) + q.close() + + from core.chunker.processor import ChecksumProcessor + worker = Worker("w-0", q, ChecksumProcessor(), max_retries=0) + results = worker.run() + + assert len(results) == 3 + assert all(r.success for r in results) + + def test_retry_on_failure(self, make_chunk): + """Worker retries on processor failure.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + proc = FailNTimesProcessor(fail_count=2) + worker = Worker("w-0", q, proc, max_retries=3) + results = worker.run() + + assert len(results) == 1 + assert results[0].success is True + assert results[0].retries == 2 + assert proc.call_count == 3 # 2 failures + 1 success + + def test_max_retries_exceeded(self, make_chunk): + """Worker gives up after max retries.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + worker = Worker("w-0", q, AlwaysFailProcessor(), max_retries=2) + results = worker.run() + + assert len(results) == 1 + assert results[0].success is False + assert results[0].error is not None + assert worker.error_count == 1 + + def test_worker_id_on_results(self, make_chunk): + """Worker stamps its ID on results.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + from core.chunker.processor import ChecksumProcessor + worker = Worker("worker-7", q, ChecksumProcessor()) + results = worker.run() + + assert results[0].worker_id == "worker-7" + + def test_event_callback(self, make_chunk): + """Worker emits events via callback.""" + q = ChunkQueue(maxsize=5) + q.put(make_chunk(0)) + q.close() + + events = [] + callback = MagicMock(side_effect=lambda t, d: events.append((t, d))) + + from core.chunker.processor import ChecksumProcessor + worker = Worker("w-0", q, ChecksumProcessor(), event_callback=callback) + worker.run() + + event_types = [e[0] for e in events] + assert "worker_status" in event_types + assert "chunk_processing" in event_types + assert "chunk_done" in event_types + + def test_processed_count(self, make_chunk): + """Worker tracks processed count.""" + q = ChunkQueue(maxsize=10) + for i in range(5): + q.put(make_chunk(i)) + q.close() + + from core.chunker.processor import ChecksumProcessor + worker = Worker("w-0", q, ChecksumProcessor()) + worker.run() + + assert worker.processed_count == 5 diff --git a/ui/chunker/index.html b/ui/chunker/index.html new file mode 100644 index 0000000..f333563 --- /dev/null +++ b/ui/chunker/index.html @@ -0,0 +1,12 @@ + + + + + + MPR Chunker Pipeline + + +
+ + + diff --git a/ui/chunker/package-lock.json b/ui/chunker/package-lock.json new file mode 100644 index 0000000..6633c19 --- /dev/null +++ b/ui/chunker/package-lock.json @@ -0,0 +1,1729 @@ +{ + "name": "mpr-chunker", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "mpr-chunker", + "version": "0.1.0", + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", + "@vitejs/plugin-react": "^4.2.0", + "typescript": "^5.3.0", + "vite": "^5.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", + "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", + "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.28.6", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", + "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", + "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.28.6", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", + "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", + "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.28.6", + "@babel/helper-validator-identifier": "^7.28.5", + "@babel/traverse": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", + "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", + "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.0.tgz", + "integrity": "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.27.1.tgz", + "integrity": "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.27.1.tgz", + "integrity": "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", + "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", + "integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz", + "integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz", + "integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz", + "integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz", + "integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz", + "integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz", + "integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz", + "integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz", + "integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz", + "integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz", + "integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz", + "integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz", + "integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz", + "integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz", + "integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz", + "integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz", + "integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz", + "integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz", + "integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz", + "integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz", + "integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz", + "integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz", + "integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz", + "integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz", + "integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.28", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz", + "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.7", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.7.tgz", + "integrity": "sha512-1ghYO3HnxGec0TCGBXiDLVns4eCSx4zJpxnHrlqFQajmhfKMQBzUGDdkMK7fUW7PTHTeLf+j87aTuKuuwWzMGw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001778", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001778.tgz", + "integrity": "sha512-PN7uxFL+ExFJO61aVmP1aIEG4i9whQd4eoSCebav62UwDyp5OHh06zN4jqKSMePVgxHifCw1QJxdRkA1Pisekg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.313", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.313.tgz", + "integrity": "sha512-QBMrTWEf00GXZmJyx2lbYD45jpI3TUFnNIzJ5BBc8piGUDwMPa1GV6HJWTZVvY/eiN3fSopl7NRbgGp9sZ9LTA==", + "dev": true, + "license": "ISC" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.36", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", + "integrity": "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.8", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", + "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", + "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.59.0", + "@rollup/rollup-android-arm64": "4.59.0", + "@rollup/rollup-darwin-arm64": "4.59.0", + "@rollup/rollup-darwin-x64": "4.59.0", + "@rollup/rollup-freebsd-arm64": "4.59.0", + "@rollup/rollup-freebsd-x64": "4.59.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", + "@rollup/rollup-linux-arm-musleabihf": "4.59.0", + "@rollup/rollup-linux-arm64-gnu": "4.59.0", + "@rollup/rollup-linux-arm64-musl": "4.59.0", + "@rollup/rollup-linux-loong64-gnu": "4.59.0", + "@rollup/rollup-linux-loong64-musl": "4.59.0", + "@rollup/rollup-linux-ppc64-gnu": "4.59.0", + "@rollup/rollup-linux-ppc64-musl": "4.59.0", + "@rollup/rollup-linux-riscv64-gnu": "4.59.0", + "@rollup/rollup-linux-riscv64-musl": "4.59.0", + "@rollup/rollup-linux-s390x-gnu": "4.59.0", + "@rollup/rollup-linux-x64-gnu": "4.59.0", + "@rollup/rollup-linux-x64-musl": "4.59.0", + "@rollup/rollup-openbsd-x64": "4.59.0", + "@rollup/rollup-openharmony-arm64": "4.59.0", + "@rollup/rollup-win32-arm64-msvc": "4.59.0", + "@rollup/rollup-win32-ia32-msvc": "4.59.0", + "@rollup/rollup-win32-x64-gnu": "4.59.0", + "@rollup/rollup-win32-x64-msvc": "4.59.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/ui/chunker/package.json b/ui/chunker/package.json new file mode 100644 index 0000000..548d863 --- /dev/null +++ b/ui/chunker/package.json @@ -0,0 +1,22 @@ +{ + "name": "mpr-chunker", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", + "@vitejs/plugin-react": "^4.2.0", + "typescript": "^5.3.0", + "vite": "^5.0.0" + } +} diff --git a/ui/chunker/src/App.css b/ui/chunker/src/App.css new file mode 100644 index 0000000..c4a0381 --- /dev/null +++ b/ui/chunker/src/App.css @@ -0,0 +1,735 @@ +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, + "Fira Code", monospace, sans-serif; + background: #0f0f0f; + color: #e0e0e0; + font-size: 14px; +} + +/* ---- Layout ---- */ + +.app { + display: flex; + flex-direction: column; + height: 100vh; +} + +.header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1.25rem; + background: #1a1a1a; + border-bottom: 1px solid #2a2a2a; +} + +.header h1 { + font-size: 1.1rem; + font-weight: 600; + letter-spacing: -0.01em; +} + +.connection-status { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; + color: #666; +} + +.dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: #555; +} + +.dot.connected { + background: #10b981; + box-shadow: 0 0 6px #10b981; +} + +.error-banner { + padding: 0.5rem 1.25rem; + background: #7f1d1d; + color: #fca5a5; + font-size: 0.85rem; +} + +.layout { + display: flex; + flex: 1; + overflow: hidden; +} + +.sidebar { + width: 300px; + background: #141414; + border-right: 1px solid #2a2a2a; + overflow-y: auto; +} + +.main { + flex: 1; + overflow-y: auto; + padding: 1rem; + display: flex; + flex-direction: column; + gap: 1rem; +} + +.main-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; +} + +.main-left, +.main-right { + display: flex; + flex-direction: column; + gap: 1rem; +} + +/* ---- Panel shared ---- */ + +.panel-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.75rem; +} + +.panel-header h2 { + font-size: 0.85rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: #888; +} + +.badge-row { + display: flex; + gap: 0.25rem; +} + +/* ---- Topic Badge ---- */ + +.topic-badge { + display: inline-flex; + align-items: center; + gap: 0.25rem; + padding: 0.15rem 0.5rem; + font-size: 0.65rem; + background: #1e293b; + border: 1px solid #334155; + border-radius: 12px; + cursor: pointer; + transition: all 0.2s; + flex-shrink: 0; +} + +.topic-badge:hover { + border-color: #3b82f6; +} + +.topic-badge.expanded { + flex-direction: column; + align-items: flex-start; + border-radius: 8px; + padding: 0.5rem; + position: relative; + z-index: 10; + background: #1e293b; +} + +.topic-number { + color: #3b82f6; + font-weight: 700; +} + +.topic-title { + color: #94a3b8; +} + +.topic-detail { + margin-top: 0.25rem; + font-size: 0.7rem; + line-height: 1.4; +} + +.topic-detail p { + color: #cbd5e1; + margin-bottom: 0.25rem; +} + +.topic-detail code { + color: #10b981; + font-size: 0.65rem; +} + +/* ---- Asset List ---- */ + +.scan-button { + padding: 0.25rem 0.5rem; + font-size: 0.7rem; + background: #1e293b; + color: #94a3b8; + border: 1px solid #334155; + border-radius: 4px; + cursor: pointer; + transition: all 0.2s; +} + +.scan-button:hover:not(:disabled) { + background: #334155; + color: #e0e0e0; +} + +.scan-button:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.asset-list { + list-style: none; + max-height: 200px; + overflow-y: auto; + margin-bottom: 0.75rem; +} + +.asset-item { + padding: 0.4rem 0.5rem; + cursor: pointer; + border-left: 2px solid transparent; + transition: all 0.15s; + display: flex; + flex-direction: column; + gap: 0.1rem; +} + +.asset-item:hover { + background: #1a1a1a; +} + +.asset-item.selected { + background: #1e293b; + border-left-color: #3b82f6; +} + +.asset-filename { + font-size: 0.8rem; + color: #e0e0e0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.asset-meta { + font-size: 0.65rem; + color: #555; +} + +.asset-empty { + font-size: 0.8rem; + color: #444; + padding: 0.75rem 0.5rem; + text-align: center; +} + +.selected-asset-info { + padding: 0.5rem; + background: #1e293b; + border: 1px solid #334155; + border-radius: 4px; + margin-bottom: 0.75rem; +} + +.asset-detail { + display: block; + font-size: 0.8rem; + color: #e0e0e0; + font-weight: 500; +} + +.asset-detail-meta { + display: block; + font-size: 0.65rem; + color: #64748b; + margin-top: 0.15rem; +} + +/* ---- Config Panel ---- */ + +.config-panel { + padding: 1rem; +} + +.config-field { + margin-bottom: 0.75rem; +} + +.config-field label { + display: block; + font-size: 0.75rem; + color: #888; + margin-bottom: 0.25rem; +} + +.config-field .default { + color: #555; + font-style: italic; +} + +.config-field input, +.config-field select { + width: 100%; + padding: 0.4rem 0.5rem; + font-size: 0.8rem; + background: #222; + color: #e0e0e0; + border: 1px solid #333; + border-radius: 4px; +} + +.config-field input:focus, +.config-field select:focus { + outline: none; + border-color: #3b82f6; +} + +.start-button { + width: 100%; + padding: 0.5rem; + font-size: 0.85rem; + background: #10b981; + color: #000; + border: none; + border-radius: 4px; + cursor: pointer; + font-weight: 600; + margin-top: 0.5rem; + transition: background 0.2s; +} + +.start-button:hover:not(:disabled) { + background: #059669; +} + +.start-button:disabled { + background: #333; + color: #666; + cursor: not-allowed; +} + +/* ---- Pipeline Diagram ---- */ + +.pipeline-diagram { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.stage-flow { + display: flex; + align-items: center; + gap: 0; + overflow-x: auto; +} + +.stage-wrapper { + display: flex; + align-items: center; +} + +.stage { + padding: 0.5rem 0.75rem; + background: #1a1a1a; + border: 1px solid #333; + border-radius: 6px; + text-align: center; + min-width: 120px; + transition: all 0.3s; +} + +.stage.active { + border-color: #3b82f6; + background: #1e293b; + box-shadow: 0 0 12px rgba(59, 130, 246, 0.2); +} + +.stage-label { + font-size: 0.8rem; + font-weight: 600; + color: #e0e0e0; +} + +.stage-sub { + font-size: 0.65rem; + color: #666; + margin-top: 0.15rem; +} + +.stage-arrow { + width: 24px; + height: 2px; + background: #444; + position: relative; +} + +.stage-arrow::after { + content: ""; + position: absolute; + right: 0; + top: -3px; + border: 4px solid transparent; + border-left: 6px solid #444; +} + +.processor-hierarchy { + margin-top: 0.75rem; + padding-top: 0.75rem; + border-top: 1px solid #222; +} + +.hierarchy-title { + font-size: 0.7rem; + color: #666; + margin-bottom: 0.35rem; + font-style: italic; +} + +.hierarchy-children { + display: flex; + gap: 0.5rem; + flex-wrap: wrap; +} + +.hierarchy-node { + font-size: 0.7rem; + padding: 0.15rem 0.5rem; + background: #1a1a1a; + border: 1px solid #333; + border-radius: 4px; + color: #94a3b8; +} + +/* ---- Chunk Grid ---- */ + +.chunk-grid-panel { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.chunk-count { + font-size: 0.7rem; + color: #555; + font-weight: 400; +} + +.chunk-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(32px, 1fr)); + gap: 3px; + max-height: 200px; + overflow-y: auto; +} + +.chunk-cell { + aspect-ratio: 1; + display: flex; + align-items: center; + justify-content: center; + font-size: 0.55rem; + color: rgba(255, 255, 255, 0.6); + border-radius: 3px; + transition: background 0.3s; +} + +.chunk-legend { + display: flex; + gap: 0.75rem; + margin-top: 0.5rem; + flex-wrap: wrap; +} + +.legend-item { + display: flex; + align-items: center; + gap: 0.25rem; + font-size: 0.65rem; + color: #888; +} + +.legend-dot { + width: 8px; + height: 8px; + border-radius: 2px; +} + +/* ---- Worker Panel ---- */ + +.worker-panel { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.worker-cards { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.worker-card { + padding: 0.5rem 0.75rem; + background: #1a1a1a; + border: 1px solid #2a2a2a; + border-radius: 6px; +} + +.worker-header { + display: flex; + justify-content: space-between; + align-items: center; +} + +.worker-name { + font-size: 0.8rem; + font-weight: 500; +} + +.worker-state { + font-size: 0.7rem; + text-transform: uppercase; + font-weight: 600; +} + +.worker-chunk { + font-size: 0.7rem; + color: #555; + margin-top: 0.15rem; +} + +.worker-stats { + display: flex; + gap: 0.75rem; + font-size: 0.65rem; + color: #555; + margin-top: 0.25rem; +} + +.worker-empty { + font-size: 0.8rem; + color: #444; + text-align: center; + padding: 1rem; +} + +/* ---- Queue Gauge ---- */ + +.queue-gauge { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.gauge-row { + margin-bottom: 0.5rem; +} + +.gauge-label { + font-size: 0.75rem; + color: #888; + margin-bottom: 0.25rem; +} + +.gauge-value { + color: #e0e0e0; + font-weight: 600; +} + +.gauge-bar { + height: 8px; + background: #222; + border-radius: 4px; + overflow: hidden; +} + +.gauge-fill { + height: 100%; + border-radius: 4px; + transition: width 0.3s, background 0.3s; +} + +.gauge-note { + font-size: 0.65rem; + color: #555; +} + +/* ---- Stats Panel ---- */ + +.stats-panel { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.stats-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 0.5rem; +} + +.stat { + text-align: center; + padding: 0.5rem; + background: #1a1a1a; + border-radius: 6px; +} + +.stat-value { + font-size: 1.1rem; + font-weight: 700; + color: #e0e0e0; +} + +.stat-label { + font-size: 0.6rem; + color: #666; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-top: 0.15rem; +} + +.test-info { + margin-top: 0.75rem; + padding-top: 0.5rem; + border-top: 1px solid #222; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.test-badge { + font-size: 0.65rem; + padding: 0.15rem 0.4rem; + background: #10b981; + color: #000; + border-radius: 3px; + font-weight: 600; +} + +.test-note { + font-size: 0.65rem; + color: #555; +} + +/* ---- Error Log ---- */ + +.error-log { + background: #141414; + border: 1px solid #2a2a2a; + border-radius: 8px; + padding: 1rem; +} + +.error-count { + font-size: 0.7rem; + background: #7f1d1d; + color: #fca5a5; + padding: 0.1rem 0.4rem; + border-radius: 8px; + font-weight: 400; +} + +.exception-tree { + margin-bottom: 0.75rem; + padding: 0.5rem; + background: #1a1a1a; + border-radius: 6px; + font-size: 0.7rem; + font-family: "Fira Code", monospace; +} + +.tree-node { + color: #94a3b8; + padding: 0.1rem 0; +} + +.tree-node.root { + color: #f59e0b; + font-weight: 600; +} + +.tree-node.leaf { + color: #64748b; +} + +.tree-children { + padding-left: 1rem; + border-left: 1px solid #333; + margin-left: 0.5rem; +} + +.tree-grandchildren { + padding-left: 1rem; + border-left: 1px solid #333; + margin-left: 0.5rem; +} + +.error-entries { + max-height: 150px; + overflow-y: auto; +} + +.error-empty { + font-size: 0.8rem; + color: #444; + text-align: center; + padding: 0.5rem; +} + +.error-entry { + display: flex; + gap: 0.5rem; + align-items: center; + padding: 0.35rem 0; + border-bottom: 1px solid #1a1a1a; + font-size: 0.7rem; + flex-wrap: wrap; +} + +.error-type { + color: #ef4444; + font-weight: 500; +} + +.error-seq { + color: #f59e0b; +} + +.error-worker { + color: #3b82f6; +} + +.error-msg { + color: #888; + flex: 1; +} + +.error-retries { + color: #f97316; + font-size: 0.65rem; +} diff --git a/ui/chunker/src/App.tsx b/ui/chunker/src/App.tsx new file mode 100644 index 0000000..b81a895 --- /dev/null +++ b/ui/chunker/src/App.tsx @@ -0,0 +1,245 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; +import "./App.css"; +import { createChunkJob, getAssets, scanMediaFolder } from "./api"; +import { ChunkGrid } from "./components/ChunkGrid"; +import { ConfigPanel } from "./components/ConfigPanel"; +import { ErrorLog } from "./components/ErrorLog"; +import { PipelineDiagram } from "./components/PipelineDiagram"; +import { QueueGauge } from "./components/QueueGauge"; +import { StatsPanel } from "./components/StatsPanel"; +import { WorkerPanel } from "./components/WorkerPanel"; +import { useEventStream } from "./hooks/useEventStream"; +import type { + ChunkInfo, + ErrorEntry, + MediaAsset, + PipelineConfig, + PipelineStats, + WorkerInfo, +} from "./types"; + +export default function App() { + const [jobId, setJobId] = useState(null); + const [running, setRunning] = useState(false); + const [error, setError] = useState(null); + + // Asset state + const [assets, setAssets] = useState([]); + const [selectedAsset, setSelectedAsset] = useState(null); + const [scanning, setScanning] = useState(false); + + const { events, connected, done } = useEventStream(jobId); + + // Load assets on mount + useEffect(() => { + getAssets() + .then((data) => setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename)))) + .catch((e) => setError(e instanceof Error ? e.message : "Failed to load assets")); + }, []); + + const handleScan = useCallback(async () => { + setScanning(true); + setError(null); + try { + await scanMediaFolder(); + const data = await getAssets(); + setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename))); + } catch (e) { + setError(e instanceof Error ? e.message : "Scan failed"); + } finally { + setScanning(false); + } + }, []); + + // Derive state from events + const { chunks, workers, stats, errors, activeStage, queueSize } = + useMemo(() => { + const chunkMap = new Map(); + const workerMap = new Map(); + const errorList: ErrorEntry[] = []; + let totalChunks = 0; + let processed = 0; + let failed = 0; + let retries = 0; + let elapsed = 0; + let throughput = 0; + let queueSize = 0; + let stage = "pending"; + + for (const evt of events) { + if (evt.total_chunks) totalChunks = evt.total_chunks; + if (evt.processed_chunks) processed = evt.processed_chunks; + if (evt.failed_chunks) failed = evt.failed_chunks; + if (evt.elapsed) elapsed = evt.elapsed; + if (evt.throughput_mbps) throughput = evt.throughput_mbps; + if (evt.queue_size !== undefined) queueSize = evt.queue_size; + if (evt.status && evt.status !== "waiting") stage = evt.status; + + // Track chunks + if (evt.sequence !== undefined) { + const existing = chunkMap.get(evt.sequence) || { + sequence: evt.sequence, + state: "pending" as const, + }; + + if (evt.status === "chunking" || evt.status === "pending") { + existing.state = "queued"; + } else if (evt.status === "processing") { + existing.state = "processing"; + if (evt.worker_id) existing.worker_id = evt.worker_id; + } else if (evt.status === "completed") { + existing.state = "done"; + if (evt.processing_time) + existing.processing_time = evt.processing_time; + if (evt.retries) existing.retries = evt.retries; + } else if (evt.status === "failed") { + existing.state = "error"; + if (evt.error) existing.error = evt.error; + } + + if (evt.size) existing.size = evt.size; + chunkMap.set(evt.sequence, existing); + } + + // Track workers + if (evt.worker_id) { + const w = workerMap.get(evt.worker_id) || { + worker_id: evt.worker_id, + state: "idle" as const, + processed: 0, + errors: 0, + retries: 0, + }; + + if (evt.state === "processing") { + w.state = "processing"; + w.current_chunk = evt.sequence; + } else if (evt.state === "idle") { + w.state = "idle"; + w.current_chunk = undefined; + } else if (evt.state === "stopped") { + w.state = "stopped"; + } + + if (evt.success !== undefined) { + if (evt.success) w.processed++; + else w.errors++; + } + if (evt.retries) { + retries += evt.retries; + w.retries += evt.retries; + } + + workerMap.set(evt.worker_id, w); + } + + // Track errors + if (evt.error) { + errorList.push({ + timestamp: Date.now(), + sequence: evt.sequence, + worker_id: evt.worker_id, + error: evt.error, + retries: evt.retries, + event_type: evt.status || "error", + }); + } + } + + const statsObj: PipelineStats = { + total_chunks: totalChunks, + processed, + failed, + retries, + elapsed, + throughput_mbps: throughput, + queue_size: queueSize, + }; + + return { + chunks: Array.from(chunkMap.values()).sort( + (a, b) => a.sequence - b.sequence + ), + workers: Array.from(workerMap.values()), + stats: statsObj, + errors: errorList, + activeStage: stage, + queueSize, + }; + }, [events]); + + const handleStart = useCallback(async (config: PipelineConfig) => { + setError(null); + setRunning(true); + try { + const result = await createChunkJob(config); + setJobId(result.id); + } catch (e) { + setError(e instanceof Error ? e.message : "Failed to start"); + setRunning(false); + } + }, []); + + // Reset running state when done + if (done && running) { + setRunning(false); + } + + return ( +
+
+

MPR Chunker Pipeline

+
+ {jobId && ( + + )} + + {!jobId + ? "Configure and launch" + : connected + ? "Streaming" + : done + ? "Complete" + : "Connecting..."} + +
+
+ + {error &&
{error}
} + +
+ + +
+ + +
+
+ + +
+
+ + + +
+
+
+
+
+ ); +} diff --git a/ui/chunker/src/api.ts b/ui/chunker/src/api.ts new file mode 100644 index 0000000..c3d4f99 --- /dev/null +++ b/ui/chunker/src/api.ts @@ -0,0 +1,72 @@ +/** + * GraphQL API client for the chunker UI. + */ + +import type { MediaAsset } from "./types"; + +const GRAPHQL_URL = "/api/graphql"; + +async function gql(query: string, variables?: Record): Promise { + const response = await fetch(GRAPHQL_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query, variables }), + }); + + const json = await response.json(); + + if (json.errors?.length) { + throw new Error(json.errors[0].message); + } + + return json.data as T; +} + +/** Fetch all media assets. */ +export async function getAssets(): Promise { + const data = await gql<{ assets: MediaAsset[] }>(` + query { + assets { + id filename file_path status error_message file_size duration + video_codec audio_codec width height framerate bitrate + properties comments tags created_at updated_at + } + } + `); + return data.assets; +} + +/** Scan media/in/ folder for new files. */ +export async function scanMediaFolder(): Promise<{ + found: number; + registered: number; + skipped: number; + files: string[]; +}> { + const data = await gql<{ scan_media_folder: { found: number; registered: number; skipped: number; files: string[] } }>(` + mutation { + scan_media_folder { found registered skipped files } + } + `); + return data.scan_media_folder; +} + +/** Create a chunk job via GraphQL mutation. */ +export async function createChunkJob(config: { + source_asset_id: string; + chunk_duration: number; + num_workers: number; + max_retries: number; + processor_type: string; +}): Promise<{ id: string }> { + const data = await gql<{ create_chunk_job: { id: string; status: string } }>(` + mutation CreateChunkJob($input: CreateChunkJobInput!) { + create_chunk_job(input: $input) { + id + status + } + } + `, { input: config }); + + return data.create_chunk_job; +} diff --git a/ui/chunker/src/components/ChunkGrid.tsx b/ui/chunker/src/components/ChunkGrid.tsx new file mode 100644 index 0000000..1f63db7 --- /dev/null +++ b/ui/chunker/src/components/ChunkGrid.tsx @@ -0,0 +1,59 @@ +import type { ChunkInfo } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + chunks: ChunkInfo[]; + totalChunks: number; +} + +const STATE_COLORS: Record = { + pending: "#333", + queued: "#f59e0b", + processing: "#3b82f6", + done: "#10b981", + error: "#ef4444", + retry: "#f97316", +}; + +/** + * Grid of chunks colored by processing state. + * Chunks appear incrementally as the generator yields them. + * Interview Topic 3: Generators & iteration. + */ +export function ChunkGrid({ chunks, totalChunks }: Props) { + return ( +
+
+

+ Chunks{" "} + + {chunks.length} / {totalChunks || "?"} + +

+ +
+
+ {chunks.map((chunk) => ( +
+ {chunk.sequence} +
+ ))} +
+
+ {Object.entries(STATE_COLORS).map(([state, color]) => ( + + + {state} + + ))} +
+
+ ); +} diff --git a/ui/chunker/src/components/ConfigPanel.tsx b/ui/chunker/src/components/ConfigPanel.tsx new file mode 100644 index 0000000..f3792b5 --- /dev/null +++ b/ui/chunker/src/components/ConfigPanel.tsx @@ -0,0 +1,172 @@ +import { useState } from "react"; +import type { MediaAsset, PipelineConfig } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + onStart: (config: PipelineConfig) => void; + running: boolean; + assets: MediaAsset[]; + selectedAsset: MediaAsset | null; + onSelectAsset: (asset: MediaAsset) => void; + onScan: () => void; + scanning: boolean; +} + +/** + * Pipeline configuration form with file browser. + * Each parameter shows its default — Interview Topic 1: Function params & defaults. + */ +export function ConfigPanel({ + onStart, + running, + assets, + selectedAsset, + onSelectAsset, + onScan, + scanning, +}: Props) { + const [chunkDuration, setChunkDuration] = useState(10.0); + const [numWorkers, setNumWorkers] = useState(4); + const [maxRetries, setMaxRetries] = useState(3); + const [processorType, setProcessorType] = useState< + "ffmpeg" | "checksum" | "simulated_decode" | "composite" + >("ffmpeg"); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (!selectedAsset) return; + onStart({ + source_asset_id: selectedAsset.id, + chunk_duration: chunkDuration, + num_workers: numWorkers, + max_retries: maxRetries, + processor_type: processorType, + }); + }; + + const formatSize = (bytes: number | null) => { + if (!bytes) return "—"; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + }; + + const formatDuration = (seconds: number | null) => { + if (!seconds) return "—"; + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m}:${s.toString().padStart(2, "0")}`; + }; + + return ( +
+ {/* Asset Browser */} +
+

Assets

+ +
+ +
    + {assets.length === 0 ? ( +
  • No assets — click Scan Folder
    • + ) : ( + assets.map((asset) => ( +
  • onSelectAsset(asset)} + title={asset.filename} + > + {asset.filename} + + {formatSize(asset.file_size)} · {formatDuration(asset.duration)} + +
    • + )) + )} +
+ + {selectedAsset && ( +
+ {selectedAsset.filename} + + {selectedAsset.video_codec} · {selectedAsset.width}x{selectedAsset.height} · {formatDuration(selectedAsset.duration)} + +
+ )} + + {/* Pipeline Config */} +
+

Pipeline Config

+ +
+
+
+ + +
+
+ + setNumWorkers(Number(e.target.value))} + /> +
+
+ + setMaxRetries(Number(e.target.value))} + /> +
+
+ + +
+ +
+
+ ); +} diff --git a/ui/chunker/src/components/ErrorLog.tsx b/ui/chunker/src/components/ErrorLog.tsx new file mode 100644 index 0000000..421705f --- /dev/null +++ b/ui/chunker/src/components/ErrorLog.tsx @@ -0,0 +1,63 @@ +import type { ErrorEntry } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + errors: ErrorEntry[]; +} + +/** + * Error and retry event log. + * Shows exception types, retry counts, backoff delays. + * Interview Topic 7: Exception handling & resilient code. + */ +export function ErrorLog({ errors }: Props) { + return ( +
+
+

+ Errors & Retries{" "} + {errors.length} +

+ +
+
+
PipelineError
+
+
ChunkError
+
+
ChunkReadError
+
ChunkChecksumError
+
+
ProcessingError
+
+
ProcessorTimeoutError
+
ProcessorFailureError
+
+
ReassemblyError
+
+
+
+ {errors.length === 0 && ( +
No errors recorded
+ )} + {errors.map((entry, i) => ( +
+ {entry.event_type} + {entry.sequence !== undefined && ( + chunk #{entry.sequence} + )} + {entry.worker_id && ( + {entry.worker_id} + )} + {entry.error} + {entry.retries !== undefined && entry.retries > 0 && ( + + {entry.retries} retries + + )} +
+ ))} +
+
+ ); +} diff --git a/ui/chunker/src/components/PipelineDiagram.tsx b/ui/chunker/src/components/PipelineDiagram.tsx new file mode 100644 index 0000000..2f8ef8e --- /dev/null +++ b/ui/chunker/src/components/PipelineDiagram.tsx @@ -0,0 +1,50 @@ +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + activeStage: string; +} + +const STAGES = [ + { id: "chunking", label: "Chunker", sub: "File -> Chunks (generator)" }, + { id: "queued", label: "ChunkQueue", sub: "Bounded queue (backpressure)" }, + { id: "processing", label: "WorkerPool", sub: "ThreadPoolExecutor" }, + { id: "collecting", label: "ResultCollector", sub: "heapq reassembly" }, + { id: "completed", label: "PipelineResult", sub: "Aggregate stats" }, +]; + +/** + * Visual flow diagram of pipeline stages. + * Highlights the currently active stage. + * Interview Topic 4: OOP design — shows class hierarchy. + */ +export function PipelineDiagram({ activeStage }: Props) { + return ( +
+
+

Pipeline Flow

+ +
+
+ {STAGES.map((stage, i) => ( +
+
+
{stage.label}
+
{stage.sub}
+
+ {i < STAGES.length - 1 &&
} +
+ ))} +
+
+
Processor ABC
+
+ ChecksumProcessor + SimulatedDecodeProcessor + CompositeProcessor +
+
+
+ ); +} diff --git a/ui/chunker/src/components/QueueGauge.tsx b/ui/chunker/src/components/QueueGauge.tsx new file mode 100644 index 0000000..305a597 --- /dev/null +++ b/ui/chunker/src/components/QueueGauge.tsx @@ -0,0 +1,46 @@ +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + current: number; + max: number; + buffered: number; +} + +/** + * Queue fill level gauge + collector heap buffer. + * Interview Topic 5: Data structures — queue.Queue, heapq, deque. + */ +export function QueueGauge({ current, max, buffered }: Props) { + const fillPct = max > 0 ? Math.min((current / max) * 100, 100) : 0; + + return ( +
+
+

Queue & Buffer

+ +
+
+
+ Queue {current}/{max} +
+
+
80 ? "#ef4444" : "#3b82f6", + }} + /> +
+
+
+
+ Heap Buffer {buffered} +
+
+ Out-of-order results waiting for gaps to fill +
+
+
+ ); +} diff --git a/ui/chunker/src/components/StatsPanel.tsx b/ui/chunker/src/components/StatsPanel.tsx new file mode 100644 index 0000000..8dd2f24 --- /dev/null +++ b/ui/chunker/src/components/StatsPanel.tsx @@ -0,0 +1,59 @@ +import type { PipelineStats } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + stats: PipelineStats; +} + +/** + * Throughput, timing, and error stats. + * Interview Topic 6: Algorithms — throughput calculation over sliding window. + * Interview Topic 8: TDD — test count and coverage. + */ +export function StatsPanel({ stats }: Props) { + return ( +
+
+

Stats

+
+ + +
+
+
+
+
{stats.total_chunks}
+
Total Chunks
+
+
+
{stats.processed}
+
Processed
+
+
+
{stats.failed}
+
Failed
+
+
+
{stats.retries}
+
Retries
+
+
+
+ {stats.throughput_mbps.toFixed(2)} +
+
MB/s
+
+
+
{stats.elapsed.toFixed(2)}s
+
Elapsed
+
+
+
+ 64 tests + + 7 test files · pytest · parametrized + +
+
+ ); +} diff --git a/ui/chunker/src/components/TopicBadge.tsx b/ui/chunker/src/components/TopicBadge.tsx new file mode 100644 index 0000000..c21f9e9 --- /dev/null +++ b/ui/chunker/src/components/TopicBadge.tsx @@ -0,0 +1,86 @@ +import { useState } from "react"; +import type { InterviewTopic } from "../types"; + +/** + * Expandable pill badge annotating an interview topic. + * Click to expand and see description + code reference. + */ +export function TopicBadge({ topic }: { topic: InterviewTopic }) { + const [expanded, setExpanded] = useState(false); + + return ( +
setExpanded(!expanded)} + > + #{topic.number} + {topic.title} + {expanded && ( +
+

{topic.description}

+ {topic.code_ref} +
+ )} +
+ ); +} + +/** Pre-defined topics mapped to pipeline components. */ +export const TOPICS: Record = { + params: { + number: 1, + title: "Function Params & Defaults", + description: + "Each pipeline parameter has a sensible default (chunk_duration=10s, num_workers=4, max_retries=3). Tweaking them changes pipeline behavior.", + code_ref: "core/chunker/pipeline.py — Pipeline.__init__()", + }, + concurrency: { + number: 2, + title: "Concurrency (Threading)", + description: + "Workers run in a ThreadPoolExecutor. The queue coordinates work between producer and consumer threads.", + code_ref: "core/chunker/pool.py — WorkerPool, ThreadPoolExecutor", + }, + iteration: { + number: 3, + title: "Generators & Iteration", + description: + "Chunks are yielded lazily via a generator — the file is never fully loaded into memory.", + code_ref: "core/chunker/chunker.py — Chunker.chunks() generator", + }, + oop: { + number: 4, + title: "OOP Design (ABC)", + description: + "Processor is an abstract base class. ChecksumProcessor, SimulatedDecodeProcessor, and CompositeProcessor inherit from it.", + code_ref: "core/chunker/processor.py — Processor ABC hierarchy", + }, + datastructures: { + number: 5, + title: "Data Structures", + description: + "Bounded queue.Queue for backpressure, heapq min-heap for ordered reassembly, deque for sliding-window throughput.", + code_ref: "core/chunker/queue.py, collector.py, models.py", + }, + algorithms: { + number: 6, + title: "Algorithms & Sorting", + description: + "ResultCollector uses a min-heap to reassemble chunks in sequence order, even when they arrive out of order.", + code_ref: "core/chunker/collector.py — heapq-based reassembly", + }, + exceptions: { + number: 7, + title: "Exception Handling", + description: + "PipelineError hierarchy with typed exceptions. Workers retry with exponential backoff before giving up.", + code_ref: "core/chunker/exceptions.py, worker.py — retry logic", + }, + testing: { + number: 8, + title: "TDD & Unit Testing", + description: + "64 tests covering every module. Parametrized tests, fixtures, edge cases, concurrency tests.", + code_ref: "tests/chunker/ — 7 test files, pytest", + }, +}; diff --git a/ui/chunker/src/components/WorkerPanel.tsx b/ui/chunker/src/components/WorkerPanel.tsx new file mode 100644 index 0000000..8017e10 --- /dev/null +++ b/ui/chunker/src/components/WorkerPanel.tsx @@ -0,0 +1,55 @@ +import type { WorkerInfo } from "../types"; +import { TopicBadge, TOPICS } from "./TopicBadge"; + +interface Props { + workers: WorkerInfo[]; +} + +const STATE_COLORS: Record = { + idle: "#6b7280", + processing: "#3b82f6", + retry: "#f97316", + stopped: "#ef4444", +}; + +/** + * Worker thread status cards. + * Shows each worker's real-time state and which chunk it's processing. + * Interview Topic 2: Concurrency (threading). + */ +export function WorkerPanel({ workers }: Props) { + return ( +
+
+

Workers

+ +
+
+ {workers.map((w) => ( +
+
+ {w.worker_id} + + {w.state} + +
+ {w.current_chunk !== undefined && ( +
chunk #{w.current_chunk}
+ )} +
+ done: {w.processed} + err: {w.errors} + retry: {w.retries} +
+
+ ))} + {workers.length === 0 && ( +
No workers started
+ )} +
+
+ ); +} diff --git a/ui/chunker/src/hooks/useEventStream.ts b/ui/chunker/src/hooks/useEventStream.ts new file mode 100644 index 0000000..00c5d58 --- /dev/null +++ b/ui/chunker/src/hooks/useEventStream.ts @@ -0,0 +1,81 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import type { PipelineEvent } from "../types"; + +/** + * SSE hook — connects to /api/chunker/stream/{jobId} via native EventSource. + * + * Demonstrates: real-time event streaming from backend to UI. + */ +export function useEventStream(jobId: string | null) { + const [events, setEvents] = useState([]); + const [connected, setConnected] = useState(false); + const [done, setDone] = useState(false); + const esRef = useRef(null); + + const close = useCallback(() => { + if (esRef.current) { + esRef.current.close(); + esRef.current = null; + setConnected(false); + } + }, []); + + useEffect(() => { + if (!jobId) return; + + setEvents([]); + setDone(false); + + const es = new EventSource(`/api/chunker/stream/${jobId}`); + esRef.current = es; + + es.onopen = () => setConnected(true); + es.onerror = () => setConnected(false); + + const handleEvent = (eventType: string) => (e: MessageEvent) => { + try { + const data = JSON.parse(e.data) as PipelineEvent; + setEvents((prev) => [...prev, { ...data, status: eventType }]); + } catch { + // ignore parse errors + } + }; + + // Listen to all chunker event types + const eventTypes = [ + "waiting", + "pending", + "chunking", + "processing", + "collecting", + "completed", + "failed", + "cancelled", + "done", + "timeout", + ]; + + for (const type of eventTypes) { + es.addEventListener(type, handleEvent(type)); + } + + es.addEventListener("done", () => { + setDone(true); + es.close(); + setConnected(false); + }); + + es.addEventListener("timeout", () => { + setDone(true); + es.close(); + setConnected(false); + }); + + return () => { + es.close(); + esRef.current = null; + }; + }, [jobId]); + + return { events, connected, done, close }; +} diff --git a/ui/chunker/src/main.tsx b/ui/chunker/src/main.tsx new file mode 100644 index 0000000..e4b1b15 --- /dev/null +++ b/ui/chunker/src/main.tsx @@ -0,0 +1,9 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; +import App from "./App"; + +ReactDOM.createRoot(document.getElementById("app")!).render( + + + +); diff --git a/ui/chunker/src/types.ts b/ui/chunker/src/types.ts new file mode 100644 index 0000000..cbfd4de --- /dev/null +++ b/ui/chunker/src/types.ts @@ -0,0 +1,114 @@ +/** Pipeline configuration sent to the backend. */ +export interface PipelineConfig { + source_asset_id: string; + chunk_duration: number; + num_workers: number; + max_retries: number; + processor_type: "ffmpeg" | "checksum" | "simulated_decode" | "composite"; +} + +/** Media asset from the backend. */ +export interface MediaAsset { + id: string; + filename: string; + file_path: string; + status: string; + error_message: string | null; + file_size: number | null; + duration: number | null; + video_codec: string | null; + audio_codec: string | null; + width: number | null; + height: number | null; + framerate: number | null; + bitrate: number | null; + properties: Record; + comments: string; + tags: string[]; + created_at: string | null; + updated_at: string | null; +} + +/** State of an individual chunk. */ +export type ChunkState = + | "pending" + | "queued" + | "processing" + | "done" + | "error" + | "retry"; + +/** Tracked chunk in the UI grid. */ +export interface ChunkInfo { + sequence: number; + state: ChunkState; + size?: number; + worker_id?: string; + retries?: number; + processing_time?: number; + error?: string; +} + +/** Worker thread status. */ +export interface WorkerInfo { + worker_id: string; + state: "idle" | "processing" | "retry" | "stopped"; + current_chunk?: number; + processed: number; + errors: number; + retries: number; +} + +/** SSE event from the backend. */ +export interface PipelineEvent { + job_id: string; + status?: string; + progress?: number; + total_chunks?: number; + processed_chunks?: number; + failed_chunks?: number; + throughput_mbps?: number; + elapsed?: number; + error?: string; + // Chunk-level fields + sequence?: number; + size?: number; + worker_id?: string; + success?: boolean; + processing_time?: number; + retries?: number; + queue_size?: number; + // Worker-level fields + state?: string; + attempt?: number; + backoff?: number; +} + +/** Aggregate pipeline stats. */ +export interface PipelineStats { + total_chunks: number; + processed: number; + failed: number; + retries: number; + elapsed: number; + throughput_mbps: number; + queue_size: number; +} + +/** Error log entry. */ +export interface ErrorEntry { + timestamp: number; + sequence?: number; + worker_id?: string; + error: string; + retries?: number; + event_type: string; +} + +/** Interview topic for annotation badges. */ +export interface InterviewTopic { + number: number; + title: string; + description: string; + code_ref: string; +} diff --git a/ui/chunker/src/vite-env.d.ts b/ui/chunker/src/vite-env.d.ts new file mode 100644 index 0000000..11f02fe --- /dev/null +++ b/ui/chunker/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/ui/chunker/tsconfig.json b/ui/chunker/tsconfig.json new file mode 100644 index 0000000..d343a63 --- /dev/null +++ b/ui/chunker/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "module": "ESNext", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*.ts", "src/**/*.tsx"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/ui/chunker/tsconfig.node.json b/ui/chunker/tsconfig.node.json new file mode 100644 index 0000000..42872c5 --- /dev/null +++ b/ui/chunker/tsconfig.node.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/ui/chunker/vite.config.ts b/ui/chunker/vite.config.ts new file mode 100644 index 0000000..4e84de9 --- /dev/null +++ b/ui/chunker/vite.config.ts @@ -0,0 +1,21 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], + server: { + host: "0.0.0.0", + port: 5174, + allowedHosts: process.env.VITE_ALLOWED_HOSTS?.split(",") || [], + proxy: { + "/api": { + target: "http://fastapi:8702", + changeOrigin: true, + }, + "/graphql": { + target: "http://fastapi:8702", + changeOrigin: true, + }, + }, + }, +}); diff --git a/ui/timeline/.dockerignore b/ui/timeline/.dockerignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/ui/timeline/.dockerignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/ui/timeline/src/types.ts b/ui/timeline/src/types.ts index d739f37..891d029 100644 --- a/ui/timeline/src/types.ts +++ b/ui/timeline/src/types.ts @@ -6,6 +6,7 @@ export type AssetStatus = "pending" | "ready" | "error"; export type JobStatus = "pending" | "processing" | "completed" | "failed" | "cancelled"; +export type ChunkJobStatus = "pending" | "chunking" | "processing" | "collecting" | "completed" | "failed" | "cancelled"; export interface MediaAsset { id: string; @@ -73,6 +74,29 @@ export interface TranscodeJob { completed_at: string | null; } +export interface ChunkJob { + id: string; + source_asset_id: string; + chunk_duration: number; + num_workers: number; + max_retries: number; + processor_type: string; + status: ChunkJobStatus; + progress: number; + total_chunks: number; + processed_chunks: number; + failed_chunks: number; + retry_count: number; + error_message: string | null; + throughput_mbps: number | null; + elapsed_seconds: number | null; + celery_task_id: string | null; + priority: number; + created_at: string | null; + started_at: string | null; + completed_at: string | null; +} + export interface CreateJobRequest { source_asset_id: string; preset_id: string | null;