diff --git a/.gitignore b/.gitignore index ecc330a..b12d928 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,10 @@ env/ *.pot *.pyc db.sqlite3 -media/* -!media/.gitkeep +media/in/* +!media/in/.gitkeep +media/out/* +!media/out/.gitkeep # Node node_modules/ diff --git a/README.md b/README.md index 44c2ed5..c163717 100644 --- a/README.md +++ b/README.md @@ -76,43 +76,38 @@ docker compose exec django python manage.py createsuperuser ## Code Generation -Models are defined in `schema/models/` and generate: -- Django ORM models -- Pydantic schemas -- TypeScript types -- Protobuf definitions +Models are defined as dataclasses in `schema/models/` and generated via `modelgen`: +- **Django ORM** models (`--include dataclasses,enums`) +- **Pydantic** schemas (`--include dataclasses,enums`) +- **TypeScript** types (`--include dataclasses,enums,api`) +- **Protobuf** definitions (`--include grpc`) + +Each target only gets the model groups it needs via the `--include` flag. ```bash -# Regenerate all -python schema/generate.py --all - -# Or specific targets -python schema/generate.py --django -python schema/generate.py --pydantic -python schema/generate.py --typescript -python schema/generate.py --proto +# Regenerate all targets +bash ctrl/generate.sh ``` ## Media Storage -MPR stores media file paths **relative to the media root** for cloud portability. +MPR separates media into **input** (`MEDIA_IN`) and **output** (`MEDIA_OUT`) paths, each independently configurable. File paths are stored relative for cloud portability. ### Local Development -- Files: `/app/media/video.mp4` -- Stored path: `video.mp4` -- Served via: `http://mpr.local.ar/media/video.mp4` (nginx alias) +- Source files: `/app/media/in/video.mp4` +- Output files: `/app/media/out/video_h264.mp4` +- Served via: `http://mpr.local.ar/media/in/video.mp4` (nginx alias) ### AWS/Cloud Deployment -For S3 or cloud storage, set `MEDIA_BASE_URL`: +Input and output can be different buckets/locations: ```bash -MEDIA_BASE_URL=https://bucket.s3.amazonaws.com/ +MEDIA_IN=s3://source-bucket/media/ +MEDIA_OUT=s3://output-bucket/transcoded/ ``` -- Files: S3 bucket -- Stored path: `video.mp4` (same relative path) -- Served via: `https://bucket.s3.amazonaws.com/video.mp4` +**Scan Endpoint**: `POST /api/assets/scan` recursively scans `MEDIA_IN` and registers new files with relative paths. -**Scan Endpoint**: `POST /api/assets/scan` recursively scans the media folder and registers new files with relative paths. +See [docs/media-storage.md](docs/media-storage.md) for full details. ## Project Structure @@ -126,7 +121,9 @@ mpr/ ├── ctrl/ # Docker & deployment │ ├── docker-compose.yml │ └── nginx.conf -├── media/ # Media files (local storage) +├── media/ +│ ├── in/ # Source media files +│ └── out/ # Transcoded output ├── rpc/ # gRPC server & client │ └── protos/ # Protobuf definitions (generated) ├── mpr/ # Django project @@ -151,7 +148,8 @@ See `ctrl/.env.template` for all configuration options. | `GRPC_HOST` | grpc | gRPC server hostname | | `GRPC_PORT` | 50051 | gRPC server port | | `MPR_EXECUTOR` | local | Executor type (local/lambda) | -| `MEDIA_ROOT` | /app/media | Media files directory | +| `MEDIA_IN` | /app/media/in | Source media files directory | +| `MEDIA_OUT` | /app/media/out | Transcoded output directory | | `MEDIA_BASE_URL` | /media/ | Base URL for serving media (use S3 URL for cloud) | | `VITE_ALLOWED_HOSTS` | - | Comma-separated allowed hosts for Vite dev server | diff --git a/api/routes/assets.py b/api/routes/assets.py index e1a51ef..5f333c2 100644 --- a/api/routes/assets.py +++ b/api/routes/assets.py @@ -33,7 +33,7 @@ def create_asset(data: AssetCreate): # Store path relative to media root import os - media_root = Path(os.environ.get("MEDIA_ROOT", "/app/media")) + media_root = Path(os.environ.get("MEDIA_IN", "/app/media/in")) try: rel_path = str(path.relative_to(media_root)) except ValueError: @@ -111,8 +111,8 @@ def scan_media_folder(): from mpr.media_assets.models import MediaAsset - # Get media root from environment - media_root = os.environ.get("MEDIA_ROOT", "/app/media") + # Get media input folder from environment + media_root = os.environ.get("MEDIA_IN", "/app/media/in") media_path = Path(media_root) if not media_path.exists(): diff --git a/api/routes/jobs.py b/api/routes/jobs.py index 007b401..31c948b 100644 --- a/api/routes/jobs.py +++ b/api/routes/jobs.py @@ -30,9 +30,6 @@ def create_job(data: JobCreate): except MediaAsset.DoesNotExist: raise HTTPException(status_code=404, detail="Source asset not found") - if source.status != "ready": - raise HTTPException(status_code=400, detail="Source asset is not ready") - # Get preset if specified preset = None preset_snapshot = {} @@ -64,27 +61,48 @@ def create_job(data: JobCreate): status_code=400, detail="Must specify preset_id or trim_start/trim_end" ) - # Generate output filename + # Generate output filename and path + import os + from pathlib import Path + output_filename = data.output_filename if not output_filename: - from pathlib import Path - stem = Path(source.filename).stem ext = preset_snapshot.get("container", "mp4") if preset else "mp4" output_filename = f"{stem}_output.{ext}" + media_out = os.environ.get("MEDIA_OUT", "/app/media/out") + output_path = str(Path(media_out) / output_filename) + + media_in = os.environ.get("MEDIA_IN", "/app/media/in") + source_path = str(Path(media_in) / source.file_path) + # Create job job = TranscodeJob.objects.create( - source_asset=source, - preset=preset, + source_asset_id=source.id, + preset_id=preset.id if preset else None, preset_snapshot=preset_snapshot, trim_start=data.trim_start, trim_end=data.trim_end, output_filename=output_filename, + output_path=output_path, priority=data.priority or 0, ) - # TODO: Submit job via gRPC + # Dispatch to Celery + from task.tasks import run_transcode_job + + result = run_transcode_job.delay( + job_id=str(job.id), + source_path=source_path, + output_path=output_path, + preset=preset_snapshot or None, + trim_start=data.trim_start, + trim_end=data.trim_end, + duration=source.duration, + ) + job.celery_task_id = result.id + job.save(update_fields=["celery_task_id"]) return job diff --git a/api/schemas/job.py b/api/schemas/job.py index f035356..c24abe4 100644 --- a/api/schemas/job.py +++ b/api/schemas/job.py @@ -17,26 +17,19 @@ class JobStatus(str, Enum): class JobCreate(BaseSchema): - """JobCreate schema.""" + """Client-facing job creation request.""" + source_asset_id: UUID preset_id: Optional[UUID] = None - preset_snapshot: Dict[str, Any] trim_start: Optional[float] = None trim_end: Optional[float] = None - output_filename: str = "" - output_path: Optional[str] = None - output_asset_id: Optional[UUID] = None - progress: float = 0.0 - current_frame: Optional[int] = None - current_time: Optional[float] = None - speed: Optional[str] = None - celery_task_id: Optional[str] = None + output_filename: Optional[str] = None priority: int = 0 - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None + class JobUpdate(BaseSchema): """JobUpdate schema.""" + source_asset_id: Optional[UUID] = None preset_id: Optional[UUID] = None preset_snapshot: Optional[Dict[str, Any]] = None @@ -56,8 +49,10 @@ class JobUpdate(BaseSchema): started_at: Optional[datetime] = None completed_at: Optional[datetime] = None + class JobResponse(BaseSchema): """JobResponse schema.""" + id: UUID source_asset_id: UUID preset_id: Optional[UUID] = None diff --git a/ctrl/.env.template b/ctrl/.env.template index 6a4be58..5e8c5f5 100644 --- a/ctrl/.env.template +++ b/ctrl/.env.template @@ -27,5 +27,9 @@ GRPC_HOST=grpc GRPC_PORT=50051 GRPC_MAX_WORKERS=10 +# Media +MEDIA_IN=/app/media/in +MEDIA_OUT=/app/media/out + # Vite VITE_ALLOWED_HOSTS=your-domain.local diff --git a/ctrl/docker-compose.yml b/ctrl/docker-compose.yml index bdc65de..27db613 100644 --- a/ctrl/docker-compose.yml +++ b/ctrl/docker-compose.yml @@ -5,6 +5,8 @@ x-common-env: &common-env DEBUG: 1 GRPC_HOST: grpc GRPC_PORT: 50051 + MEDIA_IN: ${MEDIA_IN:-/app/media/in} + MEDIA_OUT: ${MEDIA_OUT:-/app/media/out} x-healthcheck-defaults: &healthcheck-defaults interval: 5s @@ -119,7 +121,7 @@ services: build: context: .. dockerfile: ctrl/Dockerfile - command: celery -A mpr worker -l info -Q default -c 2 + command: celery -A mpr worker -l info -Q transcode -c 2 environment: <<: *common-env MPR_EXECUTOR: local diff --git a/ctrl/generate.sh b/ctrl/generate.sh index 1f3a457..5d7ce6f 100755 --- a/ctrl/generate.sh +++ b/ctrl/generate.sh @@ -8,29 +8,33 @@ cd "$(dirname "$0")/.." echo "Generating models from schema/models..." -# Django ORM models +# Django ORM models: domain models + enums python -m modelgen from-schema \ --schema schema/models \ --output mpr/media_assets/models.py \ - --targets django + --targets django \ + --include dataclasses,enums -# Pydantic schemas for FastAPI +# Pydantic schemas for FastAPI: domain models + enums python -m modelgen from-schema \ --schema schema/models \ --output api/schemas/models.py \ - --targets pydantic + --targets pydantic \ + --include dataclasses,enums -# TypeScript types for Timeline UI +# TypeScript types for Timeline UI: domain models + enums + API types python -m modelgen from-schema \ --schema schema/models \ --output ui/timeline/src/types.ts \ - --targets typescript + --targets typescript \ + --include dataclasses,enums,api -# Protobuf for gRPC +# Protobuf for gRPC: gRPC messages + service python -m modelgen from-schema \ --schema schema/models \ --output rpc/protos/worker.proto \ - --targets proto + --targets proto \ + --include grpc # Generate gRPC stubs from proto echo "Generating gRPC stubs..." diff --git a/ctrl/nginx.conf b/ctrl/nginx.conf index d9e6671..4c252d8 100644 --- a/ctrl/nginx.conf +++ b/ctrl/nginx.conf @@ -67,9 +67,15 @@ http { proxy_set_header Host $host; } - # Media files - location /media { - alias /app/media; + # Media files - input (source) + location /media/in { + alias /app/media/in; + autoindex on; + } + + # Media files - output (transcoded) + location /media/out { + alias /app/media/out; autoindex on; } diff --git a/docs/media-storage.md b/docs/media-storage.md index 396692a..011f463 100644 --- a/docs/media-storage.md +++ b/docs/media-storage.md @@ -2,13 +2,23 @@ ## Overview -MPR stores media file paths **relative to the media root** to ensure portability between local development and cloud deployments (AWS S3, etc.). +MPR separates media into **input** and **output** paths, each independently configurable. File paths are stored **relative to their respective root** to ensure portability between local development and cloud deployments (AWS S3, etc.). ## Storage Strategy +### Input / Output Separation + +| Path | Env Var | Purpose | +|------|---------|---------| +| `MEDIA_IN` | `/app/media/in` | Source media files to process | +| `MEDIA_OUT` | `/app/media/out` | Transcoded/trimmed output files | + +These can point to different locations or even different servers/buckets in production. + ### File Path Storage - **Database**: Stores only the relative path (e.g., `videos/sample.mp4`) -- **Media Root**: Configurable base directory via `MEDIA_ROOT` env var +- **Input Root**: Configurable via `MEDIA_IN` env var +- **Output Root**: Configurable via `MEDIA_OUT` env var - **Serving**: Base URL configurable via `MEDIA_BASE_URL` env var ### Why Relative Paths? @@ -20,20 +30,26 @@ MPR stores media file paths **relative to the media root** to ensure portability ### Configuration ```bash -MEDIA_ROOT=/app/media +MEDIA_IN=/app/media/in +MEDIA_OUT=/app/media/out ``` ### File Structure ``` /app/media/ -├── video1.mp4 -├── video2.mp4 -└── subfolder/ - └── video3.mp4 +├── in/ # Source files +│ ├── video1.mp4 +│ ├── video2.mp4 +│ └── subfolder/ +│ └── video3.mp4 +└── out/ # Transcoded output + ├── video1_h264.mp4 + └── video2_trimmed.mp4 ``` ### Database Storage ``` +# Source assets (scanned from media/in) filename: video1.mp4 file_path: video1.mp4 @@ -42,25 +58,31 @@ file_path: subfolder/video3.mp4 ``` ### URL Serving -- Nginx serves via `location /media { alias /app/media; }` -- Frontend accesses: `http://mpr.local.ar/media/video1.mp4` -- Video player: `