Compare commits
18 Commits
65c3055de6
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 5ceb8172ea | |||
| 3eeedebb15 | |||
| eaaf2ad60c | |||
| 4e9d731cff | |||
| dbbaad5b94 | |||
| 2ac31083e5 | |||
| f481fa6cbe | |||
| cc1a1b9953 | |||
| da1ff62877 | |||
| 9cead74fb3 | |||
| 72e4113529 | |||
| 8f5d407e0e | |||
| e642908abb | |||
| 013587d108 | |||
| 2cf6c89fbb | |||
| daabd15c19 | |||
| 2e6ed4e37a | |||
| 68622bd6b1 |
30
.dockerignore
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
# Python
|
||||||
|
.venv/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
*.egg-info/
|
||||||
|
.pytest_cache/
|
||||||
|
|
||||||
|
# Node
|
||||||
|
node_modules/
|
||||||
|
ui/*/node_modules/
|
||||||
|
ui/*/dist/
|
||||||
|
|
||||||
|
# Media (9.8GB — mounted via volume, never needed in image)
|
||||||
|
media/
|
||||||
|
|
||||||
|
# Git
|
||||||
|
.git/
|
||||||
|
|
||||||
|
# IDE / OS
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
*.swp
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# Docker
|
||||||
|
ctrl/docker-compose.yml
|
||||||
|
|
||||||
|
# Docs
|
||||||
|
docs/
|
||||||
|
*.md
|
||||||
6
.gitignore
vendored
@@ -17,8 +17,10 @@ env/
|
|||||||
*.pot
|
*.pot
|
||||||
*.pyc
|
*.pyc
|
||||||
db.sqlite3
|
db.sqlite3
|
||||||
media/*
|
media/in/*
|
||||||
!media/.gitkeep
|
!media/in/.gitkeep
|
||||||
|
media/out/*
|
||||||
|
!media/out/.gitkeep
|
||||||
|
|
||||||
# Node
|
# Node
|
||||||
node_modules/
|
node_modules/
|
||||||
|
|||||||
87
README.md
@@ -71,52 +71,73 @@ docker compose logs -f
|
|||||||
docker compose logs -f celery
|
docker compose logs -f celery
|
||||||
|
|
||||||
# Create admin user
|
# Create admin user
|
||||||
docker compose exec django python manage.py createsuperuser
|
docker compose exec django python admin/manage.py createsuperuser
|
||||||
```
|
```
|
||||||
|
|
||||||
## Code Generation
|
## Code Generation
|
||||||
|
|
||||||
Models are defined in `schema/models/` and generate:
|
Models are defined as dataclasses in `core/schema/models/` and generated via `modelgen`:
|
||||||
- Django ORM models
|
- **Django ORM** models (`--include dataclasses,enums`)
|
||||||
- Pydantic schemas
|
- **Pydantic** schemas (`--include dataclasses,enums`)
|
||||||
- TypeScript types
|
- **TypeScript** types (`--include dataclasses,enums,api`)
|
||||||
- Protobuf definitions
|
- **Protobuf** definitions (`--include grpc`)
|
||||||
|
|
||||||
|
Each target only gets the model groups it needs via the `--include` flag.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Regenerate all
|
# Regenerate all targets
|
||||||
python schema/generate.py --all
|
bash ctrl/generate.sh
|
||||||
|
|
||||||
# Or specific targets
|
|
||||||
python schema/generate.py --django
|
|
||||||
python schema/generate.py --pydantic
|
|
||||||
python schema/generate.py --typescript
|
|
||||||
python schema/generate.py --proto
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Media Storage
|
||||||
|
|
||||||
|
MPR separates media into **input** (`MEDIA_IN`) and **output** (`MEDIA_OUT`) paths, each independently configurable. File paths are stored relative for cloud portability.
|
||||||
|
|
||||||
|
### Local Development
|
||||||
|
- Source files: `/app/media/in/video.mp4`
|
||||||
|
- Output files: `/app/media/out/video_h264.mp4`
|
||||||
|
- Served via: `http://mpr.local.ar/media/in/video.mp4` (nginx alias)
|
||||||
|
|
||||||
|
### AWS/Cloud Deployment
|
||||||
|
Input and output can be different buckets/locations:
|
||||||
|
```bash
|
||||||
|
MEDIA_IN=s3://source-bucket/media/
|
||||||
|
MEDIA_OUT=s3://output-bucket/transcoded/
|
||||||
|
```
|
||||||
|
|
||||||
|
**Scan Endpoint**: `POST /api/assets/scan` recursively scans `MEDIA_IN` and registers new files with relative paths.
|
||||||
|
|
||||||
|
See [docs/media-storage.md](docs/media-storage.md) for full details.
|
||||||
|
|
||||||
## Project Structure
|
## Project Structure
|
||||||
|
|
||||||
```
|
```
|
||||||
mpr/
|
mpr/
|
||||||
├── api/ # FastAPI application
|
├── admin/ # Django project
|
||||||
│ ├── routes/ # API endpoints
|
│ ├── manage.py # Django management script
|
||||||
│ └── schemas/ # Pydantic models (generated)
|
│ └── mpr/ # Django settings & app
|
||||||
├── core/ # Core utilities
|
│ └── media_assets/# Django app
|
||||||
│ └── ffmpeg/ # FFmpeg wrappers
|
├── core/ # Core application logic
|
||||||
|
│ ├── api/ # FastAPI + GraphQL API
|
||||||
|
│ │ └── schema/ # GraphQL types (generated)
|
||||||
|
│ ├── ffmpeg/ # FFmpeg wrappers
|
||||||
|
│ ├── rpc/ # gRPC server & client
|
||||||
|
│ │ └── protos/ # Protobuf definitions (generated)
|
||||||
|
│ ├── schema/ # Source of truth
|
||||||
|
│ │ └── models/ # Dataclass definitions
|
||||||
|
│ ├── storage/ # S3/GCP/local storage backends
|
||||||
|
│ └── task/ # Celery job execution
|
||||||
|
│ ├── executor.py # Executor abstraction
|
||||||
|
│ └── tasks.py # Celery tasks
|
||||||
├── ctrl/ # Docker & deployment
|
├── ctrl/ # Docker & deployment
|
||||||
│ ├── docker-compose.yml
|
│ ├── docker-compose.yml
|
||||||
│ └── nginx.conf
|
│ └── nginx.conf
|
||||||
├── docs/ # Architecture diagrams
|
├── media/
|
||||||
├── grpc/ # gRPC server & client
|
│ ├── in/ # Source media files
|
||||||
│ └── protos/ # Protobuf definitions (generated)
|
│ └── out/ # Transcoded output
|
||||||
├── mpr/ # Django project
|
├── modelgen/ # Code generation tool
|
||||||
│ └── media_assets/ # Django app
|
└── ui/ # Frontend
|
||||||
├── schema/ # Source of truth
|
└── timeline/ # React app
|
||||||
│ └── models/ # Dataclass definitions
|
|
||||||
├── ui/ # Frontend
|
|
||||||
│ └── timeline/ # React app
|
|
||||||
└── worker/ # Job execution
|
|
||||||
├── executor.py # Executor abstraction
|
|
||||||
└── tasks.py # Celery tasks
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Environment Variables
|
## Environment Variables
|
||||||
@@ -130,6 +151,10 @@ See `ctrl/.env.template` for all configuration options.
|
|||||||
| `GRPC_HOST` | grpc | gRPC server hostname |
|
| `GRPC_HOST` | grpc | gRPC server hostname |
|
||||||
| `GRPC_PORT` | 50051 | gRPC server port |
|
| `GRPC_PORT` | 50051 | gRPC server port |
|
||||||
| `MPR_EXECUTOR` | local | Executor type (local/lambda) |
|
| `MPR_EXECUTOR` | local | Executor type (local/lambda) |
|
||||||
|
| `MEDIA_IN` | /app/media/in | Source media files directory |
|
||||||
|
| `MEDIA_OUT` | /app/media/out | Transcoded output directory |
|
||||||
|
| `MEDIA_BASE_URL` | /media/ | Base URL for serving media (use S3 URL for cloud) |
|
||||||
|
| `VITE_ALLOWED_HOSTS` | - | Comma-separated allowed hosts for Vite dev server |
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ import sys
|
|||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""Run administrative tasks."""
|
"""Run administrative tasks."""
|
||||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mpr.settings')
|
# Ensure project root is on sys.path
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'admin.mpr.settings')
|
||||||
try:
|
try:
|
||||||
from django.core.management import execute_from_command_line
|
from django.core.management import execute_from_command_line
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
@@ -11,6 +11,6 @@ import os
|
|||||||
|
|
||||||
from django.core.asgi import get_asgi_application
|
from django.core.asgi import get_asgi_application
|
||||||
|
|
||||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mpr.settings')
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'admin.mpr.settings')
|
||||||
|
|
||||||
application = get_asgi_application()
|
application = get_asgi_application()
|
||||||
@@ -2,8 +2,9 @@ import os
|
|||||||
|
|
||||||
from celery import Celery
|
from celery import Celery
|
||||||
|
|
||||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mpr.settings")
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "admin.mpr.settings")
|
||||||
|
|
||||||
app = Celery("mpr")
|
app = Celery("mpr")
|
||||||
app.config_from_object("django.conf:settings", namespace="CELERY")
|
app.config_from_object("django.conf:settings", namespace="CELERY")
|
||||||
app.autodiscover_tasks()
|
app.autodiscover_tasks()
|
||||||
|
app.autodiscover_tasks(["core.task"])
|
||||||
@@ -3,5 +3,6 @@ from django.apps import AppConfig
|
|||||||
|
|
||||||
class MediaAssetsConfig(AppConfig):
|
class MediaAssetsConfig(AppConfig):
|
||||||
default_auto_field = "django.db.models.BigAutoField"
|
default_auto_field = "django.db.models.BigAutoField"
|
||||||
name = "mpr.media_assets"
|
name = "admin.mpr.media_assets"
|
||||||
|
label = "media_assets"
|
||||||
verbose_name = "Media Assets"
|
verbose_name = "Media Assets"
|
||||||
@@ -4,10 +4,10 @@ from pathlib import Path
|
|||||||
|
|
||||||
from django.core.management.base import BaseCommand
|
from django.core.management.base import BaseCommand
|
||||||
|
|
||||||
from mpr.media_assets.models import TranscodePreset
|
from admin.mpr.media_assets.models import TranscodePreset
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent.parent.parent))
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent.parent.parent.parent))
|
||||||
from schema.models import BUILTIN_PRESETS
|
from core.schema.models import BUILTIN_PRESETS
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
@@ -1,8 +1,7 @@
|
|||||||
# Generated by Django 6.0.1 on 2026-02-01 15:13
|
# Generated by Django 4.2.29 on 2026-03-13 04:04
|
||||||
|
|
||||||
import django.db.models.deletion
|
|
||||||
import uuid
|
|
||||||
from django.db import migrations, models
|
from django.db import migrations, models
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
@@ -13,47 +12,21 @@ class Migration(migrations.Migration):
|
|||||||
]
|
]
|
||||||
|
|
||||||
operations = [
|
operations = [
|
||||||
migrations.CreateModel(
|
|
||||||
name='TranscodePreset',
|
|
||||||
fields=[
|
|
||||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
|
||||||
('name', models.CharField(max_length=100, unique=True)),
|
|
||||||
('description', models.TextField(blank=True, default='')),
|
|
||||||
('is_builtin', models.BooleanField(default=False)),
|
|
||||||
('container', models.CharField(default='mp4', max_length=20)),
|
|
||||||
('video_codec', models.CharField(default='libx264', max_length=50)),
|
|
||||||
('video_bitrate', models.CharField(blank=True, max_length=20, null=True)),
|
|
||||||
('video_crf', models.IntegerField(blank=True, null=True)),
|
|
||||||
('video_preset', models.CharField(blank=True, max_length=20, null=True)),
|
|
||||||
('resolution', models.CharField(blank=True, max_length=20, null=True)),
|
|
||||||
('framerate', models.FloatField(blank=True, null=True)),
|
|
||||||
('audio_codec', models.CharField(default='aac', max_length=50)),
|
|
||||||
('audio_bitrate', models.CharField(blank=True, max_length=20, null=True)),
|
|
||||||
('audio_channels', models.IntegerField(blank=True, null=True)),
|
|
||||||
('audio_samplerate', models.IntegerField(blank=True, null=True)),
|
|
||||||
('extra_args', models.JSONField(blank=True, default=list)),
|
|
||||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
|
||||||
('updated_at', models.DateTimeField(auto_now=True)),
|
|
||||||
],
|
|
||||||
options={
|
|
||||||
'ordering': ['name'],
|
|
||||||
},
|
|
||||||
),
|
|
||||||
migrations.CreateModel(
|
migrations.CreateModel(
|
||||||
name='MediaAsset',
|
name='MediaAsset',
|
||||||
fields=[
|
fields=[
|
||||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||||
('filename', models.CharField(max_length=500)),
|
('filename', models.CharField(max_length=500)),
|
||||||
('file_path', models.CharField(max_length=1000)),
|
('file_path', models.CharField(max_length=1000)),
|
||||||
('status', models.CharField(choices=[('pending', 'Pending Probe'), ('ready', 'Ready'), ('error', 'Error')], default='pending', max_length=20)),
|
('status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('error', 'Error')], default='pending', max_length=20)),
|
||||||
('error_message', models.TextField(blank=True, null=True)),
|
('error_message', models.TextField(blank=True, default='')),
|
||||||
('file_size', models.BigIntegerField(blank=True, null=True)),
|
('file_size', models.BigIntegerField(blank=True, null=True)),
|
||||||
('duration', models.FloatField(blank=True, null=True)),
|
('duration', models.FloatField(blank=True, default=None, null=True)),
|
||||||
('video_codec', models.CharField(blank=True, max_length=50, null=True)),
|
('video_codec', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
('audio_codec', models.CharField(blank=True, max_length=50, null=True)),
|
('audio_codec', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
('width', models.IntegerField(blank=True, null=True)),
|
('width', models.IntegerField(blank=True, default=None, null=True)),
|
||||||
('height', models.IntegerField(blank=True, null=True)),
|
('height', models.IntegerField(blank=True, default=None, null=True)),
|
||||||
('framerate', models.FloatField(blank=True, null=True)),
|
('framerate', models.FloatField(blank=True, default=None, null=True)),
|
||||||
('bitrate', models.BigIntegerField(blank=True, null=True)),
|
('bitrate', models.BigIntegerField(blank=True, null=True)),
|
||||||
('properties', models.JSONField(blank=True, default=dict)),
|
('properties', models.JSONField(blank=True, default=dict)),
|
||||||
('comments', models.TextField(blank=True, default='')),
|
('comments', models.TextField(blank=True, default='')),
|
||||||
@@ -63,36 +36,61 @@ class Migration(migrations.Migration):
|
|||||||
],
|
],
|
||||||
options={
|
options={
|
||||||
'ordering': ['-created_at'],
|
'ordering': ['-created_at'],
|
||||||
'indexes': [models.Index(fields=['status'], name='media_asset_status_9ea2f2_idx'), models.Index(fields=['created_at'], name='media_asset_created_368039_idx')],
|
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
migrations.CreateModel(
|
migrations.CreateModel(
|
||||||
name='TranscodeJob',
|
name='TranscodeJob',
|
||||||
fields=[
|
fields=[
|
||||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||||
|
('source_asset_id', models.UUIDField()),
|
||||||
|
('preset_id', models.UUIDField(blank=True, null=True)),
|
||||||
('preset_snapshot', models.JSONField(blank=True, default=dict)),
|
('preset_snapshot', models.JSONField(blank=True, default=dict)),
|
||||||
('trim_start', models.FloatField(blank=True, null=True)),
|
('trim_start', models.FloatField(blank=True, default=None, null=True)),
|
||||||
('trim_end', models.FloatField(blank=True, null=True)),
|
('trim_end', models.FloatField(blank=True, default=None, null=True)),
|
||||||
('output_filename', models.CharField(max_length=500)),
|
('output_filename', models.CharField(max_length=500)),
|
||||||
('output_path', models.CharField(blank=True, max_length=1000, null=True)),
|
('output_path', models.CharField(blank=True, max_length=1000, null=True)),
|
||||||
|
('output_asset_id', models.UUIDField(blank=True, null=True)),
|
||||||
('status', models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('completed', 'Completed'), ('failed', 'Failed'), ('cancelled', 'Cancelled')], default='pending', max_length=20)),
|
('status', models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('completed', 'Completed'), ('failed', 'Failed'), ('cancelled', 'Cancelled')], default='pending', max_length=20)),
|
||||||
('progress', models.FloatField(default=0.0)),
|
('progress', models.FloatField(default=0.0)),
|
||||||
('current_frame', models.IntegerField(blank=True, null=True)),
|
('current_frame', models.IntegerField(blank=True, default=None, null=True)),
|
||||||
('current_time', models.FloatField(blank=True, null=True)),
|
('current_time', models.FloatField(blank=True, default=None, null=True)),
|
||||||
('speed', models.CharField(blank=True, max_length=20, null=True)),
|
('speed', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
('error_message', models.TextField(blank=True, null=True)),
|
('error_message', models.TextField(blank=True, default='')),
|
||||||
('celery_task_id', models.CharField(blank=True, max_length=100, null=True)),
|
('celery_task_id', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
|
('execution_arn', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
('priority', models.IntegerField(default=0)),
|
('priority', models.IntegerField(default=0)),
|
||||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||||
('started_at', models.DateTimeField(blank=True, null=True)),
|
('started_at', models.DateTimeField(blank=True, null=True)),
|
||||||
('completed_at', models.DateTimeField(blank=True, null=True)),
|
('completed_at', models.DateTimeField(blank=True, null=True)),
|
||||||
('output_asset', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='source_jobs', to='media_assets.mediaasset')),
|
|
||||||
('source_asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcode_jobs', to='media_assets.mediaasset')),
|
|
||||||
('preset', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='jobs', to='media_assets.transcodepreset')),
|
|
||||||
],
|
],
|
||||||
options={
|
options={
|
||||||
'ordering': ['priority', 'created_at'],
|
'ordering': ['-created_at'],
|
||||||
'indexes': [models.Index(fields=['status', 'priority'], name='media_asset_status_e6ac18_idx'), models.Index(fields=['created_at'], name='media_asset_created_ba3a46_idx'), models.Index(fields=['celery_task_id'], name='media_asset_celery__81a88e_idx')],
|
},
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='TranscodePreset',
|
||||||
|
fields=[
|
||||||
|
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||||
|
('name', models.CharField(max_length=255)),
|
||||||
|
('description', models.TextField(blank=True, default='')),
|
||||||
|
('is_builtin', models.BooleanField(default=False)),
|
||||||
|
('container', models.CharField(max_length=255)),
|
||||||
|
('video_codec', models.CharField(max_length=255)),
|
||||||
|
('video_bitrate', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
|
('video_crf', models.IntegerField(blank=True, default=None, null=True)),
|
||||||
|
('video_preset', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
|
('resolution', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
|
('framerate', models.FloatField(blank=True, default=None, null=True)),
|
||||||
|
('audio_codec', models.CharField(max_length=255)),
|
||||||
|
('audio_bitrate', models.CharField(blank=True, max_length=255, null=True)),
|
||||||
|
('audio_channels', models.IntegerField(blank=True, default=None, null=True)),
|
||||||
|
('audio_samplerate', models.IntegerField(blank=True, default=None, null=True)),
|
||||||
|
('extra_args', models.JSONField(blank=True, default=list)),
|
||||||
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||||
|
('updated_at', models.DateTimeField(auto_now=True)),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'ordering': ['-created_at'],
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
@@ -25,7 +25,7 @@ class MediaAsset(models.Model):
|
|||||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||||
filename = models.CharField(max_length=500)
|
filename = models.CharField(max_length=500)
|
||||||
file_path = models.CharField(max_length=1000)
|
file_path = models.CharField(max_length=1000)
|
||||||
status = models.CharField(max_length=20, choices=Status.choices, default=Status.PENDING)
|
status = models.CharField(max_length=20, choices=AssetStatus.choices, default=AssetStatus.PENDING)
|
||||||
error_message = models.TextField(blank=True, default='')
|
error_message = models.TextField(blank=True, default='')
|
||||||
file_size = models.BigIntegerField(null=True, blank=True)
|
file_size = models.BigIntegerField(null=True, blank=True)
|
||||||
duration = models.FloatField(null=True, blank=True, default=None)
|
duration = models.FloatField(null=True, blank=True, default=None)
|
||||||
@@ -89,13 +89,14 @@ class TranscodeJob(models.Model):
|
|||||||
output_filename = models.CharField(max_length=500)
|
output_filename = models.CharField(max_length=500)
|
||||||
output_path = models.CharField(max_length=1000, null=True, blank=True)
|
output_path = models.CharField(max_length=1000, null=True, blank=True)
|
||||||
output_asset_id = models.UUIDField(null=True, blank=True)
|
output_asset_id = models.UUIDField(null=True, blank=True)
|
||||||
status = models.CharField(max_length=20, choices=Status.choices, default=Status.PENDING)
|
status = models.CharField(max_length=20, choices=JobStatus.choices, default=JobStatus.PENDING)
|
||||||
progress = models.FloatField(default=0.0)
|
progress = models.FloatField(default=0.0)
|
||||||
current_frame = models.IntegerField(null=True, blank=True, default=None)
|
current_frame = models.IntegerField(null=True, blank=True, default=None)
|
||||||
current_time = models.FloatField(null=True, blank=True, default=None)
|
current_time = models.FloatField(null=True, blank=True, default=None)
|
||||||
speed = models.CharField(max_length=255, null=True, blank=True)
|
speed = models.CharField(max_length=255, null=True, blank=True)
|
||||||
error_message = models.TextField(blank=True, default='')
|
error_message = models.TextField(blank=True, default='')
|
||||||
celery_task_id = models.CharField(max_length=255, null=True, blank=True)
|
celery_task_id = models.CharField(max_length=255, null=True, blank=True)
|
||||||
|
execution_arn = models.CharField(max_length=255, null=True, blank=True)
|
||||||
priority = models.IntegerField(default=0)
|
priority = models.IntegerField(default=0)
|
||||||
created_at = models.DateTimeField(auto_now_add=True)
|
created_at = models.DateTimeField(auto_now_add=True)
|
||||||
started_at = models.DateTimeField(null=True, blank=True)
|
started_at = models.DateTimeField(null=True, blank=True)
|
||||||
@@ -7,7 +7,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
import environ
|
import environ
|
||||||
|
|
||||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||||
|
|
||||||
env = environ.Env(
|
env = environ.Env(
|
||||||
DEBUG=(bool, False),
|
DEBUG=(bool, False),
|
||||||
@@ -27,7 +27,7 @@ INSTALLED_APPS = [
|
|||||||
"django.contrib.sessions",
|
"django.contrib.sessions",
|
||||||
"django.contrib.messages",
|
"django.contrib.messages",
|
||||||
"django.contrib.staticfiles",
|
"django.contrib.staticfiles",
|
||||||
"mpr.media_assets",
|
"admin.mpr.media_assets",
|
||||||
]
|
]
|
||||||
|
|
||||||
MIDDLEWARE = [
|
MIDDLEWARE = [
|
||||||
@@ -40,7 +40,7 @@ MIDDLEWARE = [
|
|||||||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||||
]
|
]
|
||||||
|
|
||||||
ROOT_URLCONF = "mpr.urls"
|
ROOT_URLCONF = "admin.mpr.urls"
|
||||||
|
|
||||||
TEMPLATES = [
|
TEMPLATES = [
|
||||||
{
|
{
|
||||||
@@ -57,7 +57,7 @@ TEMPLATES = [
|
|||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
WSGI_APPLICATION = "mpr.wsgi.application"
|
WSGI_APPLICATION = "admin.mpr.wsgi.application"
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
DATABASE_URL = env("DATABASE_URL", default="sqlite:///db.sqlite3")
|
DATABASE_URL = env("DATABASE_URL", default="sqlite:///db.sqlite3")
|
||||||
@@ -11,6 +11,6 @@ import os
|
|||||||
|
|
||||||
from django.core.wsgi import get_wsgi_application
|
from django.core.wsgi import get_wsgi_application
|
||||||
|
|
||||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mpr.settings')
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'admin.mpr.settings')
|
||||||
|
|
||||||
application = get_wsgi_application()
|
application = get_wsgi_application()
|
||||||
54
api/deps.py
@@ -1,54 +0,0 @@
|
|||||||
"""
|
|
||||||
FastAPI dependencies.
|
|
||||||
|
|
||||||
Provides database sessions, settings, and common dependencies.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
from functools import lru_cache
|
|
||||||
from typing import Generator
|
|
||||||
|
|
||||||
import django
|
|
||||||
from django.conf import settings as django_settings
|
|
||||||
|
|
||||||
# Initialize Django
|
|
||||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mpr.settings")
|
|
||||||
django.setup()
|
|
||||||
|
|
||||||
from mpr.media_assets.models import MediaAsset, TranscodeJob, TranscodePreset
|
|
||||||
|
|
||||||
|
|
||||||
@lru_cache
|
|
||||||
def get_settings():
|
|
||||||
"""Get Django settings."""
|
|
||||||
return django_settings
|
|
||||||
|
|
||||||
|
|
||||||
def get_asset(asset_id: str) -> MediaAsset:
|
|
||||||
"""Get asset by ID or raise 404."""
|
|
||||||
from fastapi import HTTPException
|
|
||||||
|
|
||||||
try:
|
|
||||||
return MediaAsset.objects.get(id=asset_id)
|
|
||||||
except MediaAsset.DoesNotExist:
|
|
||||||
raise HTTPException(status_code=404, detail="Asset not found")
|
|
||||||
|
|
||||||
|
|
||||||
def get_preset(preset_id: str) -> TranscodePreset:
|
|
||||||
"""Get preset by ID or raise 404."""
|
|
||||||
from fastapi import HTTPException
|
|
||||||
|
|
||||||
try:
|
|
||||||
return TranscodePreset.objects.get(id=preset_id)
|
|
||||||
except TranscodePreset.DoesNotExist:
|
|
||||||
raise HTTPException(status_code=404, detail="Preset not found")
|
|
||||||
|
|
||||||
|
|
||||||
def get_job(job_id: str) -> TranscodeJob:
|
|
||||||
"""Get job by ID or raise 404."""
|
|
||||||
from fastapi import HTTPException
|
|
||||||
|
|
||||||
try:
|
|
||||||
return TranscodeJob.objects.get(id=job_id)
|
|
||||||
except TranscodeJob.DoesNotExist:
|
|
||||||
raise HTTPException(status_code=404, detail="Job not found")
|
|
||||||
56
api/main.py
@@ -1,56 +0,0 @@
|
|||||||
"""
|
|
||||||
MPR FastAPI Application
|
|
||||||
|
|
||||||
Main entry point for the REST API.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
# Add project root to path
|
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
||||||
|
|
||||||
# Initialize Django before importing models
|
|
||||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mpr.settings")
|
|
||||||
|
|
||||||
import django
|
|
||||||
|
|
||||||
django.setup()
|
|
||||||
|
|
||||||
from fastapi import FastAPI
|
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
|
||||||
|
|
||||||
from api.routes import assets_router, jobs_router, presets_router, system_router
|
|
||||||
|
|
||||||
app = FastAPI(
|
|
||||||
title="MPR API",
|
|
||||||
description="Media Processor REST API",
|
|
||||||
version="0.1.0",
|
|
||||||
docs_url="/docs",
|
|
||||||
redoc_url="/redoc",
|
|
||||||
)
|
|
||||||
|
|
||||||
# CORS
|
|
||||||
app.add_middleware(
|
|
||||||
CORSMiddleware,
|
|
||||||
allow_origins=["http://mpr.local.ar", "http://localhost:5173"],
|
|
||||||
allow_credentials=True,
|
|
||||||
allow_methods=["*"],
|
|
||||||
allow_headers=["*"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Routes - all under /api prefix
|
|
||||||
app.include_router(system_router, prefix="/api")
|
|
||||||
app.include_router(assets_router, prefix="/api")
|
|
||||||
app.include_router(presets_router, prefix="/api")
|
|
||||||
app.include_router(jobs_router, prefix="/api")
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
|
||||||
def root():
|
|
||||||
"""API root."""
|
|
||||||
return {
|
|
||||||
"name": "MPR API",
|
|
||||||
"version": "0.1.0",
|
|
||||||
"docs": "/docs",
|
|
||||||
}
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
"""API Routes."""
|
|
||||||
|
|
||||||
from .assets import router as assets_router
|
|
||||||
from .jobs import router as jobs_router
|
|
||||||
from .presets import router as presets_router
|
|
||||||
from .system import router as system_router
|
|
||||||
|
|
||||||
__all__ = ["assets_router", "jobs_router", "presets_router", "system_router"]
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
"""
|
|
||||||
Asset endpoints - media file registration and metadata.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Optional
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
||||||
|
|
||||||
from api.deps import get_asset
|
|
||||||
from api.schemas import AssetCreate, AssetResponse, AssetUpdate
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/assets", tags=["assets"])
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/", response_model=AssetResponse, status_code=201)
|
|
||||||
def create_asset(data: AssetCreate):
|
|
||||||
"""
|
|
||||||
Register a media file as an asset.
|
|
||||||
|
|
||||||
The file must exist on disk. A probe task will be queued
|
|
||||||
to extract metadata asynchronously.
|
|
||||||
"""
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from mpr.media_assets.models import MediaAsset
|
|
||||||
|
|
||||||
# Validate file exists
|
|
||||||
path = Path(data.file_path)
|
|
||||||
if not path.exists():
|
|
||||||
raise HTTPException(status_code=400, detail="File not found")
|
|
||||||
|
|
||||||
# Create asset
|
|
||||||
asset = MediaAsset.objects.create(
|
|
||||||
filename=data.filename or path.name,
|
|
||||||
file_path=str(path.absolute()),
|
|
||||||
file_size=path.stat().st_size,
|
|
||||||
)
|
|
||||||
|
|
||||||
# TODO: Queue probe task via gRPC/Celery
|
|
||||||
|
|
||||||
return asset
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/", response_model=list[AssetResponse])
|
|
||||||
def list_assets(
|
|
||||||
status: Optional[str] = Query(None, description="Filter by status"),
|
|
||||||
limit: int = Query(50, ge=1, le=100),
|
|
||||||
offset: int = Query(0, ge=0),
|
|
||||||
):
|
|
||||||
"""List assets with optional filtering."""
|
|
||||||
from mpr.media_assets.models import MediaAsset
|
|
||||||
|
|
||||||
qs = MediaAsset.objects.all()
|
|
||||||
|
|
||||||
if status:
|
|
||||||
qs = qs.filter(status=status)
|
|
||||||
|
|
||||||
return list(qs[offset : offset + limit])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{asset_id}", response_model=AssetResponse)
|
|
||||||
def get_asset_detail(asset_id: UUID, asset=Depends(get_asset)):
|
|
||||||
"""Get asset details."""
|
|
||||||
return asset
|
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/{asset_id}", response_model=AssetResponse)
|
|
||||||
def update_asset(asset_id: UUID, data: AssetUpdate, asset=Depends(get_asset)):
|
|
||||||
"""Update asset metadata (comments, tags)."""
|
|
||||||
update_fields = []
|
|
||||||
|
|
||||||
if data.comments is not None:
|
|
||||||
asset.comments = data.comments
|
|
||||||
update_fields.append("comments")
|
|
||||||
|
|
||||||
if data.tags is not None:
|
|
||||||
asset.tags = data.tags
|
|
||||||
update_fields.append("tags")
|
|
||||||
|
|
||||||
if update_fields:
|
|
||||||
asset.save(update_fields=update_fields)
|
|
||||||
|
|
||||||
return asset
|
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{asset_id}", status_code=204)
|
|
||||||
def delete_asset(asset_id: UUID, asset=Depends(get_asset)):
|
|
||||||
"""Delete an asset."""
|
|
||||||
asset.delete()
|
|
||||||
@@ -1,160 +0,0 @@
|
|||||||
"""
|
|
||||||
Job endpoints - transcode/trim job management.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
from typing import Optional
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
||||||
|
|
||||||
from api.deps import get_asset, get_job, get_preset
|
|
||||||
from api.schemas import JobCreate, JobResponse
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/jobs", tags=["jobs"])
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/", response_model=JobResponse, status_code=201)
|
|
||||||
def create_job(data: JobCreate):
|
|
||||||
"""
|
|
||||||
Create a transcode or trim job.
|
|
||||||
|
|
||||||
- With preset_id: Full transcode using preset settings
|
|
||||||
- Without preset_id but with trim_start/end: Trim only (stream copy)
|
|
||||||
"""
|
|
||||||
from mpr.media_assets.models import MediaAsset, TranscodeJob, TranscodePreset
|
|
||||||
|
|
||||||
# Get source asset
|
|
||||||
try:
|
|
||||||
source = MediaAsset.objects.get(id=data.source_asset_id)
|
|
||||||
except MediaAsset.DoesNotExist:
|
|
||||||
raise HTTPException(status_code=404, detail="Source asset not found")
|
|
||||||
|
|
||||||
if source.status != "ready":
|
|
||||||
raise HTTPException(status_code=400, detail="Source asset is not ready")
|
|
||||||
|
|
||||||
# Get preset if specified
|
|
||||||
preset = None
|
|
||||||
preset_snapshot = {}
|
|
||||||
if data.preset_id:
|
|
||||||
try:
|
|
||||||
preset = TranscodePreset.objects.get(id=data.preset_id)
|
|
||||||
# Snapshot preset at job creation time
|
|
||||||
preset_snapshot = {
|
|
||||||
"name": preset.name,
|
|
||||||
"container": preset.container,
|
|
||||||
"video_codec": preset.video_codec,
|
|
||||||
"video_bitrate": preset.video_bitrate,
|
|
||||||
"video_crf": preset.video_crf,
|
|
||||||
"video_preset": preset.video_preset,
|
|
||||||
"resolution": preset.resolution,
|
|
||||||
"framerate": preset.framerate,
|
|
||||||
"audio_codec": preset.audio_codec,
|
|
||||||
"audio_bitrate": preset.audio_bitrate,
|
|
||||||
"audio_channels": preset.audio_channels,
|
|
||||||
"audio_samplerate": preset.audio_samplerate,
|
|
||||||
"extra_args": preset.extra_args,
|
|
||||||
}
|
|
||||||
except TranscodePreset.DoesNotExist:
|
|
||||||
raise HTTPException(status_code=404, detail="Preset not found")
|
|
||||||
|
|
||||||
# Validate trim-only job
|
|
||||||
if not preset and not data.trim_start and not data.trim_end:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=400, detail="Must specify preset_id or trim_start/trim_end"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Generate output filename
|
|
||||||
output_filename = data.output_filename
|
|
||||||
if not output_filename:
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
stem = Path(source.filename).stem
|
|
||||||
ext = preset_snapshot.get("container", "mp4") if preset else "mp4"
|
|
||||||
output_filename = f"{stem}_output.{ext}"
|
|
||||||
|
|
||||||
# Create job
|
|
||||||
job = TranscodeJob.objects.create(
|
|
||||||
source_asset=source,
|
|
||||||
preset=preset,
|
|
||||||
preset_snapshot=preset_snapshot,
|
|
||||||
trim_start=data.trim_start,
|
|
||||||
trim_end=data.trim_end,
|
|
||||||
output_filename=output_filename,
|
|
||||||
priority=data.priority or 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
# TODO: Submit job via gRPC
|
|
||||||
|
|
||||||
return job
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/", response_model=list[JobResponse])
|
|
||||||
def list_jobs(
|
|
||||||
status: Optional[str] = Query(None, description="Filter by status"),
|
|
||||||
source_asset_id: Optional[UUID] = Query(None),
|
|
||||||
limit: int = Query(50, ge=1, le=100),
|
|
||||||
offset: int = Query(0, ge=0),
|
|
||||||
):
|
|
||||||
"""List jobs with optional filtering."""
|
|
||||||
from mpr.media_assets.models import TranscodeJob
|
|
||||||
|
|
||||||
qs = TranscodeJob.objects.all()
|
|
||||||
|
|
||||||
if status:
|
|
||||||
qs = qs.filter(status=status)
|
|
||||||
if source_asset_id:
|
|
||||||
qs = qs.filter(source_asset_id=source_asset_id)
|
|
||||||
|
|
||||||
return list(qs[offset : offset + limit])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{job_id}", response_model=JobResponse)
|
|
||||||
def get_job_detail(job_id: UUID, job=Depends(get_job)):
|
|
||||||
"""Get job details including progress."""
|
|
||||||
return job
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{job_id}/progress")
|
|
||||||
def get_job_progress(job_id: UUID, job=Depends(get_job)):
|
|
||||||
"""Get real-time job progress."""
|
|
||||||
return {
|
|
||||||
"job_id": str(job.id),
|
|
||||||
"status": job.status,
|
|
||||||
"progress": job.progress,
|
|
||||||
"current_frame": job.current_frame,
|
|
||||||
"current_time": job.current_time,
|
|
||||||
"speed": job.speed,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{job_id}/cancel", response_model=JobResponse)
|
|
||||||
def cancel_job(job_id: UUID, job=Depends(get_job)):
|
|
||||||
"""Cancel a pending or processing job."""
|
|
||||||
if job.status not in ("pending", "processing"):
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=400, detail=f"Cannot cancel job with status: {job.status}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# TODO: Cancel via gRPC
|
|
||||||
|
|
||||||
job.status = "cancelled"
|
|
||||||
job.save(update_fields=["status"])
|
|
||||||
|
|
||||||
return job
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{job_id}/retry", response_model=JobResponse)
|
|
||||||
def retry_job(job_id: UUID, job=Depends(get_job)):
|
|
||||||
"""Retry a failed job."""
|
|
||||||
if job.status != "failed":
|
|
||||||
raise HTTPException(status_code=400, detail="Only failed jobs can be retried")
|
|
||||||
|
|
||||||
job.status = "pending"
|
|
||||||
job.progress = 0
|
|
||||||
job.error_message = None
|
|
||||||
job.save(update_fields=["status", "progress", "error_message"])
|
|
||||||
|
|
||||||
# TODO: Resubmit via gRPC
|
|
||||||
|
|
||||||
return job
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
"""
|
|
||||||
Preset endpoints - transcode configuration templates.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
|
||||||
|
|
||||||
from api.deps import get_preset
|
|
||||||
from api.schemas import PresetCreate, PresetResponse, PresetUpdate
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/presets", tags=["presets"])
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/", response_model=PresetResponse, status_code=201)
|
|
||||||
def create_preset(data: PresetCreate):
|
|
||||||
"""Create a custom preset."""
|
|
||||||
from mpr.media_assets.models import TranscodePreset
|
|
||||||
|
|
||||||
preset = TranscodePreset.objects.create(
|
|
||||||
name=data.name,
|
|
||||||
description=data.description or "",
|
|
||||||
container=data.container or "mp4",
|
|
||||||
video_codec=data.video_codec or "libx264",
|
|
||||||
video_bitrate=data.video_bitrate,
|
|
||||||
video_crf=data.video_crf,
|
|
||||||
video_preset=data.video_preset,
|
|
||||||
resolution=data.resolution,
|
|
||||||
framerate=data.framerate,
|
|
||||||
audio_codec=data.audio_codec or "aac",
|
|
||||||
audio_bitrate=data.audio_bitrate,
|
|
||||||
audio_channels=data.audio_channels,
|
|
||||||
audio_samplerate=data.audio_samplerate,
|
|
||||||
extra_args=data.extra_args or [],
|
|
||||||
is_builtin=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
return preset
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/", response_model=list[PresetResponse])
|
|
||||||
def list_presets(include_builtin: bool = True):
|
|
||||||
"""List all presets."""
|
|
||||||
from mpr.media_assets.models import TranscodePreset
|
|
||||||
|
|
||||||
qs = TranscodePreset.objects.all()
|
|
||||||
|
|
||||||
if not include_builtin:
|
|
||||||
qs = qs.filter(is_builtin=False)
|
|
||||||
|
|
||||||
return list(qs)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{preset_id}", response_model=PresetResponse)
|
|
||||||
def get_preset_detail(preset_id: UUID, preset=Depends(get_preset)):
|
|
||||||
"""Get preset details."""
|
|
||||||
return preset
|
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/{preset_id}", response_model=PresetResponse)
|
|
||||||
def update_preset(preset_id: UUID, data: PresetUpdate, preset=Depends(get_preset)):
|
|
||||||
"""Update a custom preset. Builtin presets cannot be modified."""
|
|
||||||
if preset.is_builtin:
|
|
||||||
raise HTTPException(status_code=403, detail="Cannot modify builtin preset")
|
|
||||||
|
|
||||||
update_fields = []
|
|
||||||
for field in [
|
|
||||||
"name",
|
|
||||||
"description",
|
|
||||||
"container",
|
|
||||||
"video_codec",
|
|
||||||
"video_bitrate",
|
|
||||||
"video_crf",
|
|
||||||
"video_preset",
|
|
||||||
"resolution",
|
|
||||||
"framerate",
|
|
||||||
"audio_codec",
|
|
||||||
"audio_bitrate",
|
|
||||||
"audio_channels",
|
|
||||||
"audio_samplerate",
|
|
||||||
"extra_args",
|
|
||||||
]:
|
|
||||||
value = getattr(data, field, None)
|
|
||||||
if value is not None:
|
|
||||||
setattr(preset, field, value)
|
|
||||||
update_fields.append(field)
|
|
||||||
|
|
||||||
if update_fields:
|
|
||||||
preset.save(update_fields=update_fields)
|
|
||||||
|
|
||||||
return preset
|
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{preset_id}", status_code=204)
|
|
||||||
def delete_preset(preset_id: UUID, preset=Depends(get_preset)):
|
|
||||||
"""Delete a custom preset. Builtin presets cannot be deleted."""
|
|
||||||
if preset.is_builtin:
|
|
||||||
raise HTTPException(status_code=403, detail="Cannot delete builtin preset")
|
|
||||||
|
|
||||||
preset.delete()
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
"""
|
|
||||||
System endpoints - health checks and FFmpeg capabilities.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from fastapi import APIRouter
|
|
||||||
|
|
||||||
from core.ffmpeg import get_decoders, get_encoders, get_formats
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/system", tags=["system"])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/health")
|
|
||||||
def health_check():
|
|
||||||
"""Health check endpoint."""
|
|
||||||
return {"status": "healthy"}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status")
|
|
||||||
def system_status():
|
|
||||||
"""System status for UI."""
|
|
||||||
return {"status": "ok", "version": "0.1.0"}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/worker")
|
|
||||||
def worker_status():
|
|
||||||
"""Worker status from gRPC."""
|
|
||||||
try:
|
|
||||||
from rpc.client import get_client
|
|
||||||
|
|
||||||
client = get_client()
|
|
||||||
status = client.get_worker_status()
|
|
||||||
if status:
|
|
||||||
return status
|
|
||||||
return {"available": False, "error": "No response from worker"}
|
|
||||||
except Exception as e:
|
|
||||||
return {"available": False, "error": str(e)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/ffmpeg/codecs")
|
|
||||||
def ffmpeg_codecs():
|
|
||||||
"""Get available FFmpeg encoders and decoders."""
|
|
||||||
return {
|
|
||||||
"encoders": get_encoders(),
|
|
||||||
"decoders": get_decoders(),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/ffmpeg/formats")
|
|
||||||
def ffmpeg_formats():
|
|
||||||
"""Get available FFmpeg muxers and demuxers."""
|
|
||||||
return get_formats()
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
"""API Schemas - GENERATED FILE"""
|
|
||||||
|
|
||||||
from .base import BaseSchema
|
|
||||||
from .asset import AssetCreate, AssetUpdate, AssetResponse
|
|
||||||
from .asset import AssetStatus
|
|
||||||
from .preset import PresetCreate, PresetUpdate, PresetResponse
|
|
||||||
from .job import JobCreate, JobUpdate, JobResponse
|
|
||||||
from .job import JobStatus
|
|
||||||
|
|
||||||
__all__ = ["BaseSchema", "AssetCreate", "AssetUpdate", "AssetResponse", "AssetStatus", "PresetCreate", "PresetUpdate", "PresetResponse", "JobCreate", "JobUpdate", "JobResponse", "JobStatus"]
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
"""MediaAsset Schemas - GENERATED FILE"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from enum import Enum
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from .base import BaseSchema
|
|
||||||
|
|
||||||
|
|
||||||
class AssetStatus(str, Enum):
|
|
||||||
PENDING = "pending"
|
|
||||||
READY = "ready"
|
|
||||||
ERROR = "error"
|
|
||||||
|
|
||||||
|
|
||||||
class AssetCreate(BaseSchema):
|
|
||||||
"""AssetCreate schema."""
|
|
||||||
filename: str
|
|
||||||
file_path: str
|
|
||||||
file_size: Optional[int] = None
|
|
||||||
duration: Optional[float] = None
|
|
||||||
video_codec: Optional[str] = None
|
|
||||||
audio_codec: Optional[str] = None
|
|
||||||
width: Optional[int] = None
|
|
||||||
height: Optional[int] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
bitrate: Optional[int] = None
|
|
||||||
properties: Dict[str, Any]
|
|
||||||
comments: str = ""
|
|
||||||
tags: List[str]
|
|
||||||
|
|
||||||
class AssetUpdate(BaseSchema):
|
|
||||||
"""AssetUpdate schema."""
|
|
||||||
filename: Optional[str] = None
|
|
||||||
file_path: Optional[str] = None
|
|
||||||
status: Optional[AssetStatus] = None
|
|
||||||
error_message: Optional[str] = None
|
|
||||||
file_size: Optional[int] = None
|
|
||||||
duration: Optional[float] = None
|
|
||||||
video_codec: Optional[str] = None
|
|
||||||
audio_codec: Optional[str] = None
|
|
||||||
width: Optional[int] = None
|
|
||||||
height: Optional[int] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
bitrate: Optional[int] = None
|
|
||||||
properties: Optional[Dict[str, Any]] = None
|
|
||||||
comments: Optional[str] = None
|
|
||||||
tags: Optional[List[str]] = None
|
|
||||||
|
|
||||||
class AssetResponse(BaseSchema):
|
|
||||||
"""AssetResponse schema."""
|
|
||||||
id: UUID
|
|
||||||
filename: str
|
|
||||||
file_path: str
|
|
||||||
status: AssetStatus = "AssetStatus.PENDING"
|
|
||||||
error_message: Optional[str] = None
|
|
||||||
file_size: Optional[int] = None
|
|
||||||
duration: Optional[float] = None
|
|
||||||
video_codec: Optional[str] = None
|
|
||||||
audio_codec: Optional[str] = None
|
|
||||||
width: Optional[int] = None
|
|
||||||
height: Optional[int] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
bitrate: Optional[int] = None
|
|
||||||
properties: Dict[str, Any]
|
|
||||||
comments: str = ""
|
|
||||||
tags: List[str]
|
|
||||||
created_at: Optional[datetime] = None
|
|
||||||
updated_at: Optional[datetime] = None
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
"""Pydantic Base Schema - GENERATED FILE"""
|
|
||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict
|
|
||||||
|
|
||||||
|
|
||||||
class BaseSchema(BaseModel):
|
|
||||||
"""Base schema with ORM mode."""
|
|
||||||
model_config = ConfigDict(from_attributes=True)
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
"""TranscodeJob Schemas - GENERATED FILE"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from enum import Enum
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from .base import BaseSchema
|
|
||||||
|
|
||||||
|
|
||||||
class JobStatus(str, Enum):
|
|
||||||
PENDING = "pending"
|
|
||||||
PROCESSING = "processing"
|
|
||||||
COMPLETED = "completed"
|
|
||||||
FAILED = "failed"
|
|
||||||
CANCELLED = "cancelled"
|
|
||||||
|
|
||||||
|
|
||||||
class JobCreate(BaseSchema):
|
|
||||||
"""JobCreate schema."""
|
|
||||||
source_asset_id: UUID
|
|
||||||
preset_id: Optional[UUID] = None
|
|
||||||
preset_snapshot: Dict[str, Any]
|
|
||||||
trim_start: Optional[float] = None
|
|
||||||
trim_end: Optional[float] = None
|
|
||||||
output_filename: str = ""
|
|
||||||
output_path: Optional[str] = None
|
|
||||||
output_asset_id: Optional[UUID] = None
|
|
||||||
progress: float = 0.0
|
|
||||||
current_frame: Optional[int] = None
|
|
||||||
current_time: Optional[float] = None
|
|
||||||
speed: Optional[str] = None
|
|
||||||
celery_task_id: Optional[str] = None
|
|
||||||
priority: int = 0
|
|
||||||
started_at: Optional[datetime] = None
|
|
||||||
completed_at: Optional[datetime] = None
|
|
||||||
|
|
||||||
class JobUpdate(BaseSchema):
|
|
||||||
"""JobUpdate schema."""
|
|
||||||
source_asset_id: Optional[UUID] = None
|
|
||||||
preset_id: Optional[UUID] = None
|
|
||||||
preset_snapshot: Optional[Dict[str, Any]] = None
|
|
||||||
trim_start: Optional[float] = None
|
|
||||||
trim_end: Optional[float] = None
|
|
||||||
output_filename: Optional[str] = None
|
|
||||||
output_path: Optional[str] = None
|
|
||||||
output_asset_id: Optional[UUID] = None
|
|
||||||
status: Optional[JobStatus] = None
|
|
||||||
progress: Optional[float] = None
|
|
||||||
current_frame: Optional[int] = None
|
|
||||||
current_time: Optional[float] = None
|
|
||||||
speed: Optional[str] = None
|
|
||||||
error_message: Optional[str] = None
|
|
||||||
celery_task_id: Optional[str] = None
|
|
||||||
priority: Optional[int] = None
|
|
||||||
started_at: Optional[datetime] = None
|
|
||||||
completed_at: Optional[datetime] = None
|
|
||||||
|
|
||||||
class JobResponse(BaseSchema):
|
|
||||||
"""JobResponse schema."""
|
|
||||||
id: UUID
|
|
||||||
source_asset_id: UUID
|
|
||||||
preset_id: Optional[UUID] = None
|
|
||||||
preset_snapshot: Dict[str, Any]
|
|
||||||
trim_start: Optional[float] = None
|
|
||||||
trim_end: Optional[float] = None
|
|
||||||
output_filename: str = ""
|
|
||||||
output_path: Optional[str] = None
|
|
||||||
output_asset_id: Optional[UUID] = None
|
|
||||||
status: JobStatus = "JobStatus.PENDING"
|
|
||||||
progress: float = 0.0
|
|
||||||
current_frame: Optional[int] = None
|
|
||||||
current_time: Optional[float] = None
|
|
||||||
speed: Optional[str] = None
|
|
||||||
error_message: Optional[str] = None
|
|
||||||
celery_task_id: Optional[str] = None
|
|
||||||
priority: int = 0
|
|
||||||
created_at: Optional[datetime] = None
|
|
||||||
started_at: Optional[datetime] = None
|
|
||||||
completed_at: Optional[datetime] = None
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
"""
|
|
||||||
Pydantic Models - GENERATED FILE
|
|
||||||
|
|
||||||
Do not edit directly. Regenerate using modelgen.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from enum import Enum
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
|
||||||
|
|
||||||
class AssetStatus(str, Enum):
|
|
||||||
PENDING = "pending"
|
|
||||||
READY = "ready"
|
|
||||||
ERROR = "error"
|
|
||||||
|
|
||||||
class JobStatus(str, Enum):
|
|
||||||
PENDING = "pending"
|
|
||||||
PROCESSING = "processing"
|
|
||||||
COMPLETED = "completed"
|
|
||||||
FAILED = "failed"
|
|
||||||
CANCELLED = "cancelled"
|
|
||||||
|
|
||||||
class MediaAsset(BaseModel):
|
|
||||||
"""A video/audio file registered in the system."""
|
|
||||||
id: UUID
|
|
||||||
filename: str
|
|
||||||
file_path: str
|
|
||||||
status: AssetStatus = "AssetStatus.PENDING"
|
|
||||||
error_message: Optional[str] = None
|
|
||||||
file_size: Optional[int] = None
|
|
||||||
duration: Optional[float] = None
|
|
||||||
video_codec: Optional[str] = None
|
|
||||||
audio_codec: Optional[str] = None
|
|
||||||
width: Optional[int] = None
|
|
||||||
height: Optional[int] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
bitrate: Optional[int] = None
|
|
||||||
properties: Dict[str, Any]
|
|
||||||
comments: str = ""
|
|
||||||
tags: List[str] = Field(default_factory=list)
|
|
||||||
created_at: Optional[datetime] = None
|
|
||||||
updated_at: Optional[datetime] = None
|
|
||||||
|
|
||||||
class TranscodePreset(BaseModel):
|
|
||||||
"""A reusable transcoding configuration (like Handbrake presets)."""
|
|
||||||
id: UUID
|
|
||||||
name: str
|
|
||||||
description: str = ""
|
|
||||||
is_builtin: bool = False
|
|
||||||
container: str = "mp4"
|
|
||||||
video_codec: str = "libx264"
|
|
||||||
video_bitrate: Optional[str] = None
|
|
||||||
video_crf: Optional[int] = None
|
|
||||||
video_preset: Optional[str] = None
|
|
||||||
resolution: Optional[str] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
audio_codec: str = "aac"
|
|
||||||
audio_bitrate: Optional[str] = None
|
|
||||||
audio_channels: Optional[int] = None
|
|
||||||
audio_samplerate: Optional[int] = None
|
|
||||||
extra_args: List[str] = Field(default_factory=list)
|
|
||||||
created_at: Optional[datetime] = None
|
|
||||||
updated_at: Optional[datetime] = None
|
|
||||||
|
|
||||||
class TranscodeJob(BaseModel):
|
|
||||||
"""A transcoding or trimming job in the queue."""
|
|
||||||
id: UUID
|
|
||||||
source_asset_id: UUID
|
|
||||||
preset_id: Optional[UUID] = None
|
|
||||||
preset_snapshot: Dict[str, Any]
|
|
||||||
trim_start: Optional[float] = None
|
|
||||||
trim_end: Optional[float] = None
|
|
||||||
output_filename: str = ""
|
|
||||||
output_path: Optional[str] = None
|
|
||||||
output_asset_id: Optional[UUID] = None
|
|
||||||
status: JobStatus = "JobStatus.PENDING"
|
|
||||||
progress: float = 0.0
|
|
||||||
current_frame: Optional[int] = None
|
|
||||||
current_time: Optional[float] = None
|
|
||||||
speed: Optional[str] = None
|
|
||||||
error_message: Optional[str] = None
|
|
||||||
celery_task_id: Optional[str] = None
|
|
||||||
priority: int = 0
|
|
||||||
created_at: Optional[datetime] = None
|
|
||||||
started_at: Optional[datetime] = None
|
|
||||||
completed_at: Optional[datetime] = None
|
|
||||||
@@ -1,66 +0,0 @@
|
|||||||
"""TranscodePreset Schemas - GENERATED FILE"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from enum import Enum
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from .base import BaseSchema
|
|
||||||
|
|
||||||
|
|
||||||
class PresetCreate(BaseSchema):
|
|
||||||
"""PresetCreate schema."""
|
|
||||||
name: str
|
|
||||||
description: str = ""
|
|
||||||
is_builtin: bool = False
|
|
||||||
container: str = "mp4"
|
|
||||||
video_codec: str = "libx264"
|
|
||||||
video_bitrate: Optional[str] = None
|
|
||||||
video_crf: Optional[int] = None
|
|
||||||
video_preset: Optional[str] = None
|
|
||||||
resolution: Optional[str] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
audio_codec: str = "aac"
|
|
||||||
audio_bitrate: Optional[str] = None
|
|
||||||
audio_channels: Optional[int] = None
|
|
||||||
audio_samplerate: Optional[int] = None
|
|
||||||
extra_args: List[str]
|
|
||||||
|
|
||||||
class PresetUpdate(BaseSchema):
|
|
||||||
"""PresetUpdate schema."""
|
|
||||||
name: Optional[str] = None
|
|
||||||
description: Optional[str] = None
|
|
||||||
is_builtin: Optional[bool] = None
|
|
||||||
container: Optional[str] = None
|
|
||||||
video_codec: Optional[str] = None
|
|
||||||
video_bitrate: Optional[str] = None
|
|
||||||
video_crf: Optional[int] = None
|
|
||||||
video_preset: Optional[str] = None
|
|
||||||
resolution: Optional[str] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
audio_codec: Optional[str] = None
|
|
||||||
audio_bitrate: Optional[str] = None
|
|
||||||
audio_channels: Optional[int] = None
|
|
||||||
audio_samplerate: Optional[int] = None
|
|
||||||
extra_args: Optional[List[str]] = None
|
|
||||||
|
|
||||||
class PresetResponse(BaseSchema):
|
|
||||||
"""PresetResponse schema."""
|
|
||||||
id: UUID
|
|
||||||
name: str
|
|
||||||
description: str = ""
|
|
||||||
is_builtin: bool = False
|
|
||||||
container: str = "mp4"
|
|
||||||
video_codec: str = "libx264"
|
|
||||||
video_bitrate: Optional[str] = None
|
|
||||||
video_crf: Optional[int] = None
|
|
||||||
video_preset: Optional[str] = None
|
|
||||||
resolution: Optional[str] = None
|
|
||||||
framerate: Optional[float] = None
|
|
||||||
audio_codec: str = "aac"
|
|
||||||
audio_bitrate: Optional[str] = None
|
|
||||||
audio_channels: Optional[int] = None
|
|
||||||
audio_samplerate: Optional[int] = None
|
|
||||||
extra_args: List[str]
|
|
||||||
created_at: Optional[datetime] = None
|
|
||||||
updated_at: Optional[datetime] = None
|
|
||||||
273
core/api/graphql.py
Normal file
@@ -0,0 +1,273 @@
|
|||||||
|
"""
|
||||||
|
GraphQL API using strawberry, served via FastAPI.
|
||||||
|
|
||||||
|
Primary API for MPR — all client interactions go through GraphQL.
|
||||||
|
Uses core.db for data access.
|
||||||
|
Types are generated from schema/ via modelgen — see api/schema/graphql.py.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import List, Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import strawberry
|
||||||
|
from strawberry.schema.config import StrawberryConfig
|
||||||
|
from strawberry.types import Info
|
||||||
|
|
||||||
|
from core.api.schema.graphql import (
|
||||||
|
CreateJobInput,
|
||||||
|
DeleteResultType,
|
||||||
|
MediaAssetType,
|
||||||
|
ScanResultType,
|
||||||
|
SystemStatusType,
|
||||||
|
TranscodeJobType,
|
||||||
|
TranscodePresetType,
|
||||||
|
UpdateAssetInput,
|
||||||
|
)
|
||||||
|
from core.storage import BUCKET_IN, list_objects
|
||||||
|
|
||||||
|
VIDEO_EXTS = {".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv", ".wmv", ".m4v"}
|
||||||
|
AUDIO_EXTS = {".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"}
|
||||||
|
MEDIA_EXTS = VIDEO_EXTS | AUDIO_EXTS
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Queries
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class Query:
|
||||||
|
@strawberry.field
|
||||||
|
def assets(
|
||||||
|
self,
|
||||||
|
info: Info,
|
||||||
|
status: Optional[str] = None,
|
||||||
|
search: Optional[str] = None,
|
||||||
|
) -> List[MediaAssetType]:
|
||||||
|
from core.db import list_assets
|
||||||
|
|
||||||
|
return list_assets(status=status, search=search)
|
||||||
|
|
||||||
|
@strawberry.field
|
||||||
|
def asset(self, info: Info, id: UUID) -> Optional[MediaAssetType]:
|
||||||
|
from core.db import get_asset
|
||||||
|
|
||||||
|
try:
|
||||||
|
return get_asset(id)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@strawberry.field
|
||||||
|
def jobs(
|
||||||
|
self,
|
||||||
|
info: Info,
|
||||||
|
status: Optional[str] = None,
|
||||||
|
source_asset_id: Optional[UUID] = None,
|
||||||
|
) -> List[TranscodeJobType]:
|
||||||
|
from core.db import list_jobs
|
||||||
|
|
||||||
|
return list_jobs(status=status, source_asset_id=source_asset_id)
|
||||||
|
|
||||||
|
@strawberry.field
|
||||||
|
def job(self, info: Info, id: UUID) -> Optional[TranscodeJobType]:
|
||||||
|
from core.db import get_job
|
||||||
|
|
||||||
|
try:
|
||||||
|
return get_job(id)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@strawberry.field
|
||||||
|
def presets(self, info: Info) -> List[TranscodePresetType]:
|
||||||
|
from core.db import list_presets
|
||||||
|
|
||||||
|
return list_presets()
|
||||||
|
|
||||||
|
@strawberry.field
|
||||||
|
def system_status(self, info: Info) -> SystemStatusType:
|
||||||
|
return SystemStatusType(status="ok", version="0.1.0")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Mutations
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class Mutation:
|
||||||
|
@strawberry.mutation
|
||||||
|
def scan_media_folder(self, info: Info) -> ScanResultType:
|
||||||
|
from core.db import create_asset, get_asset_filenames
|
||||||
|
|
||||||
|
objects = list_objects(BUCKET_IN, extensions=MEDIA_EXTS)
|
||||||
|
existing = get_asset_filenames()
|
||||||
|
|
||||||
|
registered = []
|
||||||
|
skipped = []
|
||||||
|
|
||||||
|
for obj in objects:
|
||||||
|
if obj["filename"] in existing:
|
||||||
|
skipped.append(obj["filename"])
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
create_asset(
|
||||||
|
filename=obj["filename"],
|
||||||
|
file_path=obj["key"],
|
||||||
|
file_size=obj["size"],
|
||||||
|
)
|
||||||
|
registered.append(obj["filename"])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return ScanResultType(
|
||||||
|
found=len(objects),
|
||||||
|
registered=len(registered),
|
||||||
|
skipped=len(skipped),
|
||||||
|
files=registered,
|
||||||
|
)
|
||||||
|
|
||||||
|
@strawberry.mutation
|
||||||
|
def create_job(self, info: Info, input: CreateJobInput) -> TranscodeJobType:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from core.db import create_job, get_asset, get_preset
|
||||||
|
|
||||||
|
try:
|
||||||
|
source = get_asset(input.source_asset_id)
|
||||||
|
except Exception:
|
||||||
|
raise Exception("Source asset not found")
|
||||||
|
|
||||||
|
preset = None
|
||||||
|
preset_snapshot = {}
|
||||||
|
if input.preset_id:
|
||||||
|
try:
|
||||||
|
preset = get_preset(input.preset_id)
|
||||||
|
preset_snapshot = {
|
||||||
|
"name": preset.name,
|
||||||
|
"container": preset.container,
|
||||||
|
"video_codec": preset.video_codec,
|
||||||
|
"audio_codec": preset.audio_codec,
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
raise Exception("Preset not found")
|
||||||
|
|
||||||
|
if not preset and not input.trim_start and not input.trim_end:
|
||||||
|
raise Exception("Must specify preset_id or trim_start/trim_end")
|
||||||
|
|
||||||
|
output_filename = input.output_filename
|
||||||
|
if not output_filename:
|
||||||
|
stem = Path(source.filename).stem
|
||||||
|
ext = preset_snapshot.get("container", "mp4") if preset else "mp4"
|
||||||
|
output_filename = f"{stem}_output.{ext}"
|
||||||
|
|
||||||
|
job = create_job(
|
||||||
|
source_asset_id=source.id,
|
||||||
|
preset_id=preset.id if preset else None,
|
||||||
|
preset_snapshot=preset_snapshot,
|
||||||
|
trim_start=input.trim_start,
|
||||||
|
trim_end=input.trim_end,
|
||||||
|
output_filename=output_filename,
|
||||||
|
output_path=output_filename,
|
||||||
|
priority=input.priority or 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
executor_mode = os.environ.get("MPR_EXECUTOR", "local")
|
||||||
|
if executor_mode in ("lambda", "gcp"):
|
||||||
|
from core.task.executor import get_executor
|
||||||
|
|
||||||
|
get_executor().run(
|
||||||
|
job_id=str(job.id),
|
||||||
|
source_path=source.file_path,
|
||||||
|
output_path=output_filename,
|
||||||
|
preset=preset_snapshot or None,
|
||||||
|
trim_start=input.trim_start,
|
||||||
|
trim_end=input.trim_end,
|
||||||
|
duration=source.duration,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
from core.task.tasks import run_transcode_job
|
||||||
|
|
||||||
|
result = run_transcode_job.delay(
|
||||||
|
job_id=str(job.id),
|
||||||
|
source_key=source.file_path,
|
||||||
|
output_key=output_filename,
|
||||||
|
preset=preset_snapshot or None,
|
||||||
|
trim_start=input.trim_start,
|
||||||
|
trim_end=input.trim_end,
|
||||||
|
duration=source.duration,
|
||||||
|
)
|
||||||
|
job.celery_task_id = result.id
|
||||||
|
job.save(update_fields=["celery_task_id"])
|
||||||
|
|
||||||
|
return job
|
||||||
|
|
||||||
|
@strawberry.mutation
|
||||||
|
def cancel_job(self, info: Info, id: UUID) -> TranscodeJobType:
|
||||||
|
from core.db import get_job, update_job
|
||||||
|
|
||||||
|
try:
|
||||||
|
job = get_job(id)
|
||||||
|
except Exception:
|
||||||
|
raise Exception("Job not found")
|
||||||
|
|
||||||
|
if job.status not in ("pending", "processing"):
|
||||||
|
raise Exception(f"Cannot cancel job with status: {job.status}")
|
||||||
|
|
||||||
|
return update_job(job, status="cancelled")
|
||||||
|
|
||||||
|
@strawberry.mutation
|
||||||
|
def retry_job(self, info: Info, id: UUID) -> TranscodeJobType:
|
||||||
|
from core.db import get_job, update_job
|
||||||
|
|
||||||
|
try:
|
||||||
|
job = get_job(id)
|
||||||
|
except Exception:
|
||||||
|
raise Exception("Job not found")
|
||||||
|
|
||||||
|
if job.status != "failed":
|
||||||
|
raise Exception("Only failed jobs can be retried")
|
||||||
|
|
||||||
|
return update_job(job, status="pending", progress=0, error_message=None)
|
||||||
|
|
||||||
|
@strawberry.mutation
|
||||||
|
def update_asset(self, info: Info, id: UUID, input: UpdateAssetInput) -> MediaAssetType:
|
||||||
|
from core.db import get_asset, update_asset
|
||||||
|
|
||||||
|
try:
|
||||||
|
asset = get_asset(id)
|
||||||
|
except Exception:
|
||||||
|
raise Exception("Asset not found")
|
||||||
|
|
||||||
|
fields = {}
|
||||||
|
if input.comments is not None:
|
||||||
|
fields["comments"] = input.comments
|
||||||
|
if input.tags is not None:
|
||||||
|
fields["tags"] = input.tags
|
||||||
|
|
||||||
|
if fields:
|
||||||
|
asset = update_asset(asset, **fields)
|
||||||
|
|
||||||
|
return asset
|
||||||
|
|
||||||
|
@strawberry.mutation
|
||||||
|
def delete_asset(self, info: Info, id: UUID) -> DeleteResultType:
|
||||||
|
from core.db import delete_asset, get_asset
|
||||||
|
|
||||||
|
try:
|
||||||
|
asset = get_asset(id)
|
||||||
|
delete_asset(asset)
|
||||||
|
return DeleteResultType(ok=True)
|
||||||
|
except Exception:
|
||||||
|
raise Exception("Asset not found")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Schema
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
schema = strawberry.Schema(
|
||||||
|
query=Query,
|
||||||
|
mutation=Mutation,
|
||||||
|
config=StrawberryConfig(auto_camel_case=False),
|
||||||
|
)
|
||||||
98
core/api/main.py
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
"""
|
||||||
|
MPR FastAPI Application
|
||||||
|
|
||||||
|
Serves GraphQL API and Lambda callback endpoint.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
# Add project root to path
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||||
|
|
||||||
|
# Initialize Django before importing models
|
||||||
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "admin.mpr.settings")
|
||||||
|
|
||||||
|
import django
|
||||||
|
|
||||||
|
django.setup()
|
||||||
|
|
||||||
|
from fastapi import FastAPI, Header, HTTPException
|
||||||
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
from strawberry.fastapi import GraphQLRouter
|
||||||
|
|
||||||
|
from core.api.graphql import schema as graphql_schema
|
||||||
|
|
||||||
|
CALLBACK_API_KEY = os.environ.get("CALLBACK_API_KEY", "")
|
||||||
|
|
||||||
|
app = FastAPI(
|
||||||
|
title="MPR API",
|
||||||
|
description="Media Processor — GraphQL API",
|
||||||
|
version="0.1.0",
|
||||||
|
docs_url="/docs",
|
||||||
|
redoc_url="/redoc",
|
||||||
|
)
|
||||||
|
|
||||||
|
# CORS
|
||||||
|
app.add_middleware(
|
||||||
|
CORSMiddleware,
|
||||||
|
allow_origins=["http://mpr.local.ar", "http://localhost:5173"],
|
||||||
|
allow_credentials=True,
|
||||||
|
allow_methods=["*"],
|
||||||
|
allow_headers=["*"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# GraphQL
|
||||||
|
graphql_router = GraphQLRouter(schema=graphql_schema, graphql_ide="graphiql")
|
||||||
|
app.include_router(graphql_router, prefix="/graphql")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/")
|
||||||
|
def root():
|
||||||
|
"""API root."""
|
||||||
|
return {
|
||||||
|
"name": "MPR API",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"graphql": "/graphql",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/jobs/{job_id}/callback")
|
||||||
|
def job_callback(
|
||||||
|
job_id: UUID,
|
||||||
|
payload: dict,
|
||||||
|
x_api_key: Optional[str] = Header(None),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Callback endpoint for Lambda to report job completion.
|
||||||
|
Protected by API key.
|
||||||
|
"""
|
||||||
|
if CALLBACK_API_KEY and x_api_key != CALLBACK_API_KEY:
|
||||||
|
raise HTTPException(status_code=403, detail="Invalid API key")
|
||||||
|
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from core.db import get_job, update_job
|
||||||
|
|
||||||
|
try:
|
||||||
|
job = get_job(job_id)
|
||||||
|
except Exception:
|
||||||
|
raise HTTPException(status_code=404, detail="Job not found")
|
||||||
|
|
||||||
|
status = payload.get("status", "failed")
|
||||||
|
fields = {
|
||||||
|
"status": status,
|
||||||
|
"progress": 100.0 if status == "completed" else job.progress,
|
||||||
|
}
|
||||||
|
|
||||||
|
if payload.get("error"):
|
||||||
|
fields["error_message"] = payload["error"]
|
||||||
|
|
||||||
|
if status in ("completed", "failed"):
|
||||||
|
fields["completed_at"] = timezone.now()
|
||||||
|
|
||||||
|
update_job(job, **fields)
|
||||||
|
|
||||||
|
return {"ok": True}
|
||||||
158
core/api/schema/graphql.py
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
"""
|
||||||
|
Strawberry Types - GENERATED FILE
|
||||||
|
|
||||||
|
Do not edit directly. Regenerate using modelgen.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import strawberry
|
||||||
|
from enum import Enum
|
||||||
|
from typing import List, Optional
|
||||||
|
from uuid import UUID
|
||||||
|
from datetime import datetime
|
||||||
|
from strawberry.scalars import JSON
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.enum
|
||||||
|
class AssetStatus(Enum):
|
||||||
|
PENDING = "pending"
|
||||||
|
READY = "ready"
|
||||||
|
ERROR = "error"
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.enum
|
||||||
|
class JobStatus(Enum):
|
||||||
|
PENDING = "pending"
|
||||||
|
PROCESSING = "processing"
|
||||||
|
COMPLETED = "completed"
|
||||||
|
FAILED = "failed"
|
||||||
|
CANCELLED = "cancelled"
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class MediaAssetType:
|
||||||
|
"""A video/audio file registered in the system."""
|
||||||
|
|
||||||
|
id: Optional[UUID] = None
|
||||||
|
filename: Optional[str] = None
|
||||||
|
file_path: Optional[str] = None
|
||||||
|
status: Optional[str] = None
|
||||||
|
error_message: Optional[str] = None
|
||||||
|
file_size: Optional[int] = None
|
||||||
|
duration: Optional[float] = None
|
||||||
|
video_codec: Optional[str] = None
|
||||||
|
audio_codec: Optional[str] = None
|
||||||
|
width: Optional[int] = None
|
||||||
|
height: Optional[int] = None
|
||||||
|
framerate: Optional[float] = None
|
||||||
|
bitrate: Optional[int] = None
|
||||||
|
properties: Optional[JSON] = None
|
||||||
|
comments: Optional[str] = None
|
||||||
|
tags: Optional[List[str]] = None
|
||||||
|
created_at: Optional[datetime] = None
|
||||||
|
updated_at: Optional[datetime] = None
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class TranscodePresetType:
|
||||||
|
"""A reusable transcoding configuration (like Handbrake presets)."""
|
||||||
|
|
||||||
|
id: Optional[UUID] = None
|
||||||
|
name: Optional[str] = None
|
||||||
|
description: Optional[str] = None
|
||||||
|
is_builtin: Optional[bool] = None
|
||||||
|
container: Optional[str] = None
|
||||||
|
video_codec: Optional[str] = None
|
||||||
|
video_bitrate: Optional[str] = None
|
||||||
|
video_crf: Optional[int] = None
|
||||||
|
video_preset: Optional[str] = None
|
||||||
|
resolution: Optional[str] = None
|
||||||
|
framerate: Optional[float] = None
|
||||||
|
audio_codec: Optional[str] = None
|
||||||
|
audio_bitrate: Optional[str] = None
|
||||||
|
audio_channels: Optional[int] = None
|
||||||
|
audio_samplerate: Optional[int] = None
|
||||||
|
extra_args: Optional[List[str]] = None
|
||||||
|
created_at: Optional[datetime] = None
|
||||||
|
updated_at: Optional[datetime] = None
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class TranscodeJobType:
|
||||||
|
"""A transcoding or trimming job in the queue."""
|
||||||
|
|
||||||
|
id: Optional[UUID] = None
|
||||||
|
source_asset_id: Optional[UUID] = None
|
||||||
|
preset_id: Optional[UUID] = None
|
||||||
|
preset_snapshot: Optional[JSON] = None
|
||||||
|
trim_start: Optional[float] = None
|
||||||
|
trim_end: Optional[float] = None
|
||||||
|
output_filename: Optional[str] = None
|
||||||
|
output_path: Optional[str] = None
|
||||||
|
output_asset_id: Optional[UUID] = None
|
||||||
|
status: Optional[str] = None
|
||||||
|
progress: Optional[float] = None
|
||||||
|
current_frame: Optional[int] = None
|
||||||
|
current_time: Optional[float] = None
|
||||||
|
speed: Optional[str] = None
|
||||||
|
error_message: Optional[str] = None
|
||||||
|
celery_task_id: Optional[str] = None
|
||||||
|
execution_arn: Optional[str] = None
|
||||||
|
priority: Optional[int] = None
|
||||||
|
created_at: Optional[datetime] = None
|
||||||
|
started_at: Optional[datetime] = None
|
||||||
|
completed_at: Optional[datetime] = None
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.input
|
||||||
|
class CreateJobInput:
|
||||||
|
"""Request body for creating a transcode/trim job."""
|
||||||
|
|
||||||
|
source_asset_id: UUID
|
||||||
|
preset_id: Optional[UUID] = None
|
||||||
|
trim_start: Optional[float] = None
|
||||||
|
trim_end: Optional[float] = None
|
||||||
|
output_filename: Optional[str] = None
|
||||||
|
priority: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.input
|
||||||
|
class UpdateAssetInput:
|
||||||
|
"""Request body for updating asset metadata."""
|
||||||
|
|
||||||
|
comments: Optional[str] = None
|
||||||
|
tags: Optional[List[str]] = None
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class SystemStatusType:
|
||||||
|
"""System status response."""
|
||||||
|
|
||||||
|
status: Optional[str] = None
|
||||||
|
version: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class ScanResultType:
|
||||||
|
"""Result of scanning the media input bucket."""
|
||||||
|
|
||||||
|
found: Optional[int] = None
|
||||||
|
registered: Optional[int] = None
|
||||||
|
skipped: Optional[int] = None
|
||||||
|
files: Optional[List[str]] = None
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class DeleteResultType:
|
||||||
|
"""Result of a delete operation."""
|
||||||
|
|
||||||
|
ok: Optional[bool] = None
|
||||||
|
|
||||||
|
|
||||||
|
@strawberry.type
|
||||||
|
class WorkerStatusType:
|
||||||
|
"""Worker health and capabilities."""
|
||||||
|
|
||||||
|
available: Optional[bool] = None
|
||||||
|
active_jobs: Optional[int] = None
|
||||||
|
supported_codecs: Optional[List[str]] = None
|
||||||
|
gpu_available: Optional[bool] = None
|
||||||
19
core/db/__init__.py
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
from .assets import (
|
||||||
|
create_asset,
|
||||||
|
delete_asset,
|
||||||
|
get_asset,
|
||||||
|
get_asset_filenames,
|
||||||
|
list_assets,
|
||||||
|
update_asset,
|
||||||
|
)
|
||||||
|
from .jobs import (
|
||||||
|
create_job,
|
||||||
|
get_job,
|
||||||
|
list_jobs,
|
||||||
|
update_job,
|
||||||
|
update_job_fields,
|
||||||
|
)
|
||||||
|
from .presets import (
|
||||||
|
get_preset,
|
||||||
|
list_presets,
|
||||||
|
)
|
||||||
48
core/db/assets.py
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
"""Database operations for MediaAsset."""
|
||||||
|
|
||||||
|
from typing import Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
def list_assets(status: Optional[str] = None, search: Optional[str] = None):
|
||||||
|
from admin.mpr.media_assets.models import MediaAsset
|
||||||
|
|
||||||
|
qs = MediaAsset.objects.all()
|
||||||
|
if status:
|
||||||
|
qs = qs.filter(status=status)
|
||||||
|
if search:
|
||||||
|
qs = qs.filter(filename__icontains=search)
|
||||||
|
return list(qs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_asset(id: UUID):
|
||||||
|
from admin.mpr.media_assets.models import MediaAsset
|
||||||
|
|
||||||
|
return MediaAsset.objects.get(id=id)
|
||||||
|
|
||||||
|
|
||||||
|
def get_asset_filenames() -> set[str]:
|
||||||
|
from admin.mpr.media_assets.models import MediaAsset
|
||||||
|
|
||||||
|
return set(MediaAsset.objects.values_list("filename", flat=True))
|
||||||
|
|
||||||
|
|
||||||
|
def create_asset(*, filename: str, file_path: str, file_size: int):
|
||||||
|
from admin.mpr.media_assets.models import MediaAsset
|
||||||
|
|
||||||
|
return MediaAsset.objects.create(
|
||||||
|
filename=filename,
|
||||||
|
file_path=file_path,
|
||||||
|
file_size=file_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def update_asset(asset, **fields):
|
||||||
|
for key, value in fields.items():
|
||||||
|
setattr(asset, key, value)
|
||||||
|
asset.save(update_fields=list(fields.keys()))
|
||||||
|
return asset
|
||||||
|
|
||||||
|
|
||||||
|
def delete_asset(asset):
|
||||||
|
asset.delete()
|
||||||
40
core/db/jobs.py
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
"""Database operations for TranscodeJob."""
|
||||||
|
|
||||||
|
from typing import Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
def list_jobs(status: Optional[str] = None, source_asset_id: Optional[UUID] = None):
|
||||||
|
from admin.mpr.media_assets.models import TranscodeJob
|
||||||
|
|
||||||
|
qs = TranscodeJob.objects.all()
|
||||||
|
if status:
|
||||||
|
qs = qs.filter(status=status)
|
||||||
|
if source_asset_id:
|
||||||
|
qs = qs.filter(source_asset_id=source_asset_id)
|
||||||
|
return list(qs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_job(id: UUID):
|
||||||
|
from admin.mpr.media_assets.models import TranscodeJob
|
||||||
|
|
||||||
|
return TranscodeJob.objects.get(id=id)
|
||||||
|
|
||||||
|
|
||||||
|
def create_job(**fields):
|
||||||
|
from admin.mpr.media_assets.models import TranscodeJob
|
||||||
|
|
||||||
|
return TranscodeJob.objects.create(**fields)
|
||||||
|
|
||||||
|
|
||||||
|
def update_job(job, **fields):
|
||||||
|
for key, value in fields.items():
|
||||||
|
setattr(job, key, value)
|
||||||
|
job.save(update_fields=list(fields.keys()))
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def update_job_fields(job_id, **fields):
|
||||||
|
from admin.mpr.media_assets.models import TranscodeJob
|
||||||
|
|
||||||
|
TranscodeJob.objects.filter(id=job_id).update(**fields)
|
||||||
15
core/db/presets.py
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
"""Database operations for TranscodePreset."""
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
def list_presets():
|
||||||
|
from admin.mpr.media_assets.models import TranscodePreset
|
||||||
|
|
||||||
|
return list(TranscodePreset.objects.all())
|
||||||
|
|
||||||
|
|
||||||
|
def get_preset(id: UUID):
|
||||||
|
from admin.mpr.media_assets.models import TranscodePreset
|
||||||
|
|
||||||
|
return TranscodePreset.objects.get(id=id)
|
||||||
@@ -59,7 +59,7 @@ class WorkerServicer(worker_pb2_grpc.WorkerServiceServicer):
|
|||||||
|
|
||||||
# Dispatch to Celery if available
|
# Dispatch to Celery if available
|
||||||
if self.celery_app:
|
if self.celery_app:
|
||||||
from task.tasks import run_transcode_job
|
from core.task.tasks import run_transcode_job
|
||||||
|
|
||||||
task = run_transcode_job.delay(
|
task = run_transcode_job.delay(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
@@ -201,7 +201,7 @@ def update_job_progress(
|
|||||||
"""
|
"""
|
||||||
Update job progress (called from worker tasks).
|
Update job progress (called from worker tasks).
|
||||||
|
|
||||||
This updates the in-memory state that StreamProgress reads from.
|
Updates both the in-memory gRPC state and the Django database.
|
||||||
"""
|
"""
|
||||||
if job_id in _active_jobs:
|
if job_id in _active_jobs:
|
||||||
_active_jobs[job_id].update(
|
_active_jobs[job_id].update(
|
||||||
@@ -215,6 +215,32 @@ def update_job_progress(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Update Django database
|
||||||
|
try:
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from core.db import update_job_fields
|
||||||
|
|
||||||
|
updates = {
|
||||||
|
"progress": progress,
|
||||||
|
"current_frame": current_frame,
|
||||||
|
"current_time": current_time,
|
||||||
|
"speed": str(speed),
|
||||||
|
"status": status,
|
||||||
|
}
|
||||||
|
|
||||||
|
if error:
|
||||||
|
updates["error_message"] = error
|
||||||
|
|
||||||
|
if status == "processing":
|
||||||
|
updates["started_at"] = timezone.now()
|
||||||
|
elif status in ("completed", "failed"):
|
||||||
|
updates["completed_at"] = timezone.now()
|
||||||
|
|
||||||
|
update_job_fields(job_id, **updates)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to update job {job_id} in DB: {e}")
|
||||||
|
|
||||||
|
|
||||||
def serve(port: int = None, celery_app=None) -> grpc.Server:
|
def serve(port: int = None, celery_app=None) -> grpc.Server:
|
||||||
"""
|
"""
|
||||||
@@ -4,7 +4,7 @@ MPR Schema Definitions - Source of Truth
|
|||||||
This package defines the core data models as Python dataclasses.
|
This package defines the core data models as Python dataclasses.
|
||||||
These definitions are used to generate:
|
These definitions are used to generate:
|
||||||
- Django ORM models (mpr/media_assets/models.py)
|
- Django ORM models (mpr/media_assets/models.py)
|
||||||
- Pydantic schemas (api/schemas/*.py)
|
- Pydantic schemas (api/schema/*.py)
|
||||||
- TypeScript types (ui/timeline/src/types.ts)
|
- TypeScript types (ui/timeline/src/types.ts)
|
||||||
- Protobuf definitions (grpc/protos/worker.proto)
|
- Protobuf definitions (grpc/protos/worker.proto)
|
||||||
|
|
||||||
25
core/schema/modelgen.json
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"schema": "core/schema/models",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"target": "django",
|
||||||
|
"output": "admin/mpr/media_assets/models.py",
|
||||||
|
"include": ["dataclasses", "enums"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"target": "graphene",
|
||||||
|
"output": "core/api/schema/graphql.py",
|
||||||
|
"include": ["dataclasses", "enums", "api"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"target": "typescript",
|
||||||
|
"output": "ui/timeline/src/types.ts",
|
||||||
|
"include": ["dataclasses", "enums", "api"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"target": "protobuf",
|
||||||
|
"output": "core/rpc/protos/worker.proto",
|
||||||
|
"include": ["grpc"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -5,7 +5,13 @@ This module exports all dataclasses, enums, and constants that the generator
|
|||||||
should process. Add new models here to have them included in generation.
|
should process. Add new models here to have them included in generation.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .api import CreateJobRequest, SystemStatus
|
from .api import (
|
||||||
|
CreateJobRequest,
|
||||||
|
DeleteResult,
|
||||||
|
ScanResult,
|
||||||
|
SystemStatus,
|
||||||
|
UpdateAssetRequest,
|
||||||
|
)
|
||||||
from .grpc import (
|
from .grpc import (
|
||||||
GRPC_SERVICE,
|
GRPC_SERVICE,
|
||||||
CancelRequest,
|
CancelRequest,
|
||||||
@@ -26,7 +32,14 @@ DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob]
|
|||||||
|
|
||||||
# API request/response models - generates TypeScript only (no Django)
|
# API request/response models - generates TypeScript only (no Django)
|
||||||
# WorkerStatus from grpc.py is reused here
|
# WorkerStatus from grpc.py is reused here
|
||||||
API_MODELS = [CreateJobRequest, SystemStatus, WorkerStatus]
|
API_MODELS = [
|
||||||
|
CreateJobRequest,
|
||||||
|
UpdateAssetRequest,
|
||||||
|
SystemStatus,
|
||||||
|
ScanResult,
|
||||||
|
DeleteResult,
|
||||||
|
WorkerStatus,
|
||||||
|
]
|
||||||
|
|
||||||
# Status enums - included in generated code
|
# Status enums - included in generated code
|
||||||
ENUMS = [AssetStatus, JobStatus]
|
ENUMS = [AssetStatus, JobStatus]
|
||||||
@@ -50,6 +63,9 @@ __all__ = [
|
|||||||
"TranscodeJob",
|
"TranscodeJob",
|
||||||
# API Models
|
# API Models
|
||||||
"CreateJobRequest",
|
"CreateJobRequest",
|
||||||
|
"UpdateAssetRequest",
|
||||||
|
"DeleteResult",
|
||||||
|
"ScanResult",
|
||||||
"SystemStatus",
|
"SystemStatus",
|
||||||
# Enums
|
# Enums
|
||||||
"AssetStatus",
|
"AssetStatus",
|
||||||
@@ -5,8 +5,8 @@ These are separate from the main domain models and represent
|
|||||||
the shape of data sent to/from the API endpoints.
|
the shape of data sent to/from the API endpoints.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
from typing import Optional
|
from typing import List, Optional
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
@@ -19,6 +19,7 @@ class CreateJobRequest:
|
|||||||
trim_start: Optional[float] = None # seconds
|
trim_start: Optional[float] = None # seconds
|
||||||
trim_end: Optional[float] = None # seconds
|
trim_end: Optional[float] = None # seconds
|
||||||
output_filename: Optional[str] = None
|
output_filename: Optional[str] = None
|
||||||
|
priority: int = 0
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -29,4 +30,29 @@ class SystemStatus:
|
|||||||
version: str
|
version: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ScanResult:
|
||||||
|
"""Result of scanning the media input bucket."""
|
||||||
|
|
||||||
|
found: int = 0
|
||||||
|
registered: int = 0
|
||||||
|
skipped: int = 0
|
||||||
|
files: List[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class UpdateAssetRequest:
|
||||||
|
"""Request body for updating asset metadata."""
|
||||||
|
|
||||||
|
comments: Optional[str] = None
|
||||||
|
tags: Optional[List[str]] = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DeleteResult:
|
||||||
|
"""Result of a delete operation."""
|
||||||
|
|
||||||
|
ok: bool = False
|
||||||
|
|
||||||
|
|
||||||
# Note: WorkerStatus is defined in grpc.py and reused here
|
# Note: WorkerStatus is defined in grpc.py and reused here
|
||||||
@@ -63,6 +63,7 @@ class TranscodeJob:
|
|||||||
|
|
||||||
# Worker tracking
|
# Worker tracking
|
||||||
celery_task_id: Optional[str] = None
|
celery_task_id: Optional[str] = None
|
||||||
|
execution_arn: Optional[str] = None # AWS Step Functions execution ARN
|
||||||
priority: int = 0 # Lower = higher priority
|
priority: int = 0 # Lower = higher priority
|
||||||
|
|
||||||
# Timestamps
|
# Timestamps
|
||||||
10
core/storage/__init__.py
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from .s3 import (
|
||||||
|
BUCKET_IN,
|
||||||
|
BUCKET_OUT,
|
||||||
|
download_file,
|
||||||
|
download_to_temp,
|
||||||
|
get_presigned_url,
|
||||||
|
get_s3_client,
|
||||||
|
list_objects,
|
||||||
|
upload_file,
|
||||||
|
)
|
||||||
1
core/storage/gcp.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""GCP Cloud Storage backend (placeholder)."""
|
||||||
1
core/storage/local.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""Local filesystem storage backend (placeholder)."""
|
||||||
90
core/storage/s3.py
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
"""
|
||||||
|
S3 storage layer.
|
||||||
|
|
||||||
|
Uses MinIO locally (S3-compatible) and real AWS S3 in production.
|
||||||
|
The only difference is S3_ENDPOINT_URL: set for MinIO, omit for AWS.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
BUCKET_IN = os.environ.get("S3_BUCKET_IN", "mpr-media-in")
|
||||||
|
BUCKET_OUT = os.environ.get("S3_BUCKET_OUT", "mpr-media-out")
|
||||||
|
|
||||||
|
|
||||||
|
def get_s3_client():
|
||||||
|
"""Get a boto3 S3 client. Works with both MinIO and real AWS S3."""
|
||||||
|
kwargs = {
|
||||||
|
"region_name": os.environ.get("AWS_REGION", "us-east-1"),
|
||||||
|
"config": Config(signature_version="s3v4"),
|
||||||
|
}
|
||||||
|
endpoint = os.environ.get("S3_ENDPOINT_URL")
|
||||||
|
if endpoint:
|
||||||
|
kwargs["endpoint_url"] = endpoint
|
||||||
|
kwargs["aws_access_key_id"] = os.environ.get("AWS_ACCESS_KEY_ID", "minioadmin")
|
||||||
|
kwargs["aws_secret_access_key"] = os.environ.get("AWS_SECRET_ACCESS_KEY", "minioadmin")
|
||||||
|
return boto3.client("s3", **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def list_objects(bucket: str, prefix: str = "", extensions: Optional[set] = None) -> list[dict]:
|
||||||
|
"""List objects in an S3 bucket, optionally filtered by file extension."""
|
||||||
|
s3 = get_s3_client()
|
||||||
|
objects = []
|
||||||
|
kwargs = {"Bucket": bucket, "Prefix": prefix}
|
||||||
|
|
||||||
|
while True:
|
||||||
|
response = s3.list_objects_v2(**kwargs)
|
||||||
|
for obj in response.get("Contents", []):
|
||||||
|
key = obj["Key"]
|
||||||
|
if extensions:
|
||||||
|
ext = Path(key).suffix.lower()
|
||||||
|
if ext not in extensions:
|
||||||
|
continue
|
||||||
|
objects.append({
|
||||||
|
"key": key,
|
||||||
|
"size": obj["Size"],
|
||||||
|
"filename": Path(key).name,
|
||||||
|
})
|
||||||
|
if not response.get("IsTruncated"):
|
||||||
|
break
|
||||||
|
kwargs["ContinuationToken"] = response["NextContinuationToken"]
|
||||||
|
|
||||||
|
return objects
|
||||||
|
|
||||||
|
|
||||||
|
def download_file(bucket: str, key: str, local_path: str) -> str:
|
||||||
|
"""Download a file from S3 to a local path."""
|
||||||
|
s3 = get_s3_client()
|
||||||
|
Path(local_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
s3.download_file(bucket, key, local_path)
|
||||||
|
return local_path
|
||||||
|
|
||||||
|
|
||||||
|
def download_to_temp(bucket: str, key: str) -> str:
|
||||||
|
"""Download a file from S3 to a temp file. Caller must clean up."""
|
||||||
|
ext = Path(key).suffix
|
||||||
|
fd, tmp_path = tempfile.mkstemp(suffix=ext)
|
||||||
|
os.close(fd)
|
||||||
|
download_file(bucket, key, tmp_path)
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
def upload_file(local_path: str, bucket: str, key: str) -> None:
|
||||||
|
"""Upload a local file to S3."""
|
||||||
|
s3 = get_s3_client()
|
||||||
|
s3.upload_file(local_path, bucket, key)
|
||||||
|
|
||||||
|
|
||||||
|
def get_presigned_url(bucket: str, key: str, expires: int = 3600) -> str:
|
||||||
|
"""Generate a presigned URL for an S3 object."""
|
||||||
|
s3 = get_s3_client()
|
||||||
|
return s3.generate_presigned_url(
|
||||||
|
"get_object",
|
||||||
|
Params={"Bucket": bucket, "Key": key},
|
||||||
|
ExpiresIn=expires,
|
||||||
|
)
|
||||||
@@ -110,7 +110,16 @@ class LocalExecutor(Executor):
|
|||||||
|
|
||||||
|
|
||||||
class LambdaExecutor(Executor):
|
class LambdaExecutor(Executor):
|
||||||
"""Execute jobs via AWS Lambda (future implementation)."""
|
"""Execute jobs via AWS Step Functions + Lambda."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
import boto3
|
||||||
|
|
||||||
|
region = os.environ.get("AWS_REGION", "us-east-1")
|
||||||
|
self.sfn = boto3.client("stepfunctions", region_name=region)
|
||||||
|
self.state_machine_arn = os.environ["STEP_FUNCTION_ARN"]
|
||||||
|
self.callback_url = os.environ.get("CALLBACK_URL", "")
|
||||||
|
self.callback_api_key = os.environ.get("CALLBACK_API_KEY", "")
|
||||||
|
|
||||||
def run(
|
def run(
|
||||||
self,
|
self,
|
||||||
@@ -123,14 +132,116 @@ class LambdaExecutor(Executor):
|
|||||||
duration: Optional[float] = None,
|
duration: Optional[float] = None,
|
||||||
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Execute job via AWS Lambda."""
|
"""Start a Step Functions execution for this job."""
|
||||||
raise NotImplementedError("LambdaExecutor not yet implemented")
|
import json
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"job_id": job_id,
|
||||||
|
"source_key": source_path,
|
||||||
|
"output_key": output_path,
|
||||||
|
"preset": preset,
|
||||||
|
"trim_start": trim_start,
|
||||||
|
"trim_end": trim_end,
|
||||||
|
"duration": duration,
|
||||||
|
"callback_url": self.callback_url,
|
||||||
|
"api_key": self.callback_api_key,
|
||||||
|
}
|
||||||
|
|
||||||
|
response = self.sfn.start_execution(
|
||||||
|
stateMachineArn=self.state_machine_arn,
|
||||||
|
name=f"mpr-{job_id}",
|
||||||
|
input=json.dumps(payload),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Store execution ARN on the job
|
||||||
|
execution_arn = response["executionArn"]
|
||||||
|
try:
|
||||||
|
from core.db import update_job_fields
|
||||||
|
update_job_fields(job_id, execution_arn=execution_arn)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class GCPExecutor(Executor):
|
||||||
|
"""Execute jobs via Google Cloud Run Jobs."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
from google.cloud import run_v2
|
||||||
|
|
||||||
|
self.client = run_v2.JobsClient()
|
||||||
|
self.project_id = os.environ["GCP_PROJECT_ID"]
|
||||||
|
self.region = os.environ.get("GCP_REGION", "us-central1")
|
||||||
|
self.job_name = os.environ["CLOUD_RUN_JOB"]
|
||||||
|
self.callback_url = os.environ.get("CALLBACK_URL", "")
|
||||||
|
self.callback_api_key = os.environ.get("CALLBACK_API_KEY", "")
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
job_id: str,
|
||||||
|
source_path: str,
|
||||||
|
output_path: str,
|
||||||
|
preset: Optional[Dict[str, Any]] = None,
|
||||||
|
trim_start: Optional[float] = None,
|
||||||
|
trim_end: Optional[float] = None,
|
||||||
|
duration: Optional[float] = None,
|
||||||
|
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Trigger a Cloud Run Job execution for this job."""
|
||||||
|
import json
|
||||||
|
|
||||||
|
from google.cloud import run_v2
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"job_id": job_id,
|
||||||
|
"source_key": source_path,
|
||||||
|
"output_key": output_path,
|
||||||
|
"preset": preset,
|
||||||
|
"trim_start": trim_start,
|
||||||
|
"trim_end": trim_end,
|
||||||
|
"duration": duration,
|
||||||
|
"callback_url": self.callback_url,
|
||||||
|
"api_key": self.callback_api_key,
|
||||||
|
}
|
||||||
|
|
||||||
|
job_path = (
|
||||||
|
f"projects/{self.project_id}/locations/{self.region}/jobs/{self.job_name}"
|
||||||
|
)
|
||||||
|
|
||||||
|
request = run_v2.RunJobRequest(
|
||||||
|
name=job_path,
|
||||||
|
overrides=run_v2.RunJobRequest.Overrides(
|
||||||
|
container_overrides=[
|
||||||
|
run_v2.RunJobRequest.Overrides.ContainerOverride(
|
||||||
|
env=[
|
||||||
|
run_v2.EnvVar(
|
||||||
|
name="MPR_JOB_PAYLOAD", value=json.dumps(payload)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
operation = self.client.run_job(request=request)
|
||||||
|
execution_name = operation.metadata.name
|
||||||
|
|
||||||
|
try:
|
||||||
|
from core.db import update_job_fields
|
||||||
|
|
||||||
|
update_job_fields(job_id, execution_arn=execution_name)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
# Executor registry
|
# Executor registry
|
||||||
_executors: Dict[str, type] = {
|
_executors: Dict[str, type] = {
|
||||||
"local": LocalExecutor,
|
"local": LocalExecutor,
|
||||||
"lambda": LambdaExecutor,
|
"lambda": LambdaExecutor,
|
||||||
|
"gcp": GCPExecutor,
|
||||||
}
|
}
|
||||||
|
|
||||||
_executor_instance: Optional[Executor] = None
|
_executor_instance: Optional[Executor] = None
|
||||||
121
core/task/gcp_handler.py
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
"""
|
||||||
|
Google Cloud Run Job handler for media transcoding.
|
||||||
|
|
||||||
|
Reads job payload from the MPR_JOB_PAYLOAD env var (injected by GCPExecutor),
|
||||||
|
downloads source from S3-compatible storage (GCS via HMAC + S3 API),
|
||||||
|
runs FFmpeg, uploads result, and calls back to the API.
|
||||||
|
|
||||||
|
Uses core/storage and core/ffmpeg — same modules as the Celery worker.
|
||||||
|
No cloud-provider SDK required here; storage goes through core.storage (boto3 + S3 compat).
|
||||||
|
|
||||||
|
Entry point: python -m task.gcp_handler (set as Cloud Run Job command)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from core.ffmpeg.transcode import TranscodeConfig, transcode
|
||||||
|
from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
raw = os.environ.get("MPR_JOB_PAYLOAD")
|
||||||
|
if not raw:
|
||||||
|
logger.error("MPR_JOB_PAYLOAD not set")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
event = json.loads(raw)
|
||||||
|
job_id = event["job_id"]
|
||||||
|
source_key = event["source_key"]
|
||||||
|
output_key = event["output_key"]
|
||||||
|
preset = event.get("preset")
|
||||||
|
trim_start = event.get("trim_start")
|
||||||
|
trim_end = event.get("trim_end")
|
||||||
|
duration = event.get("duration")
|
||||||
|
callback_url = event.get("callback_url", "")
|
||||||
|
api_key = event.get("api_key", "")
|
||||||
|
|
||||||
|
logger.info(f"Starting job {job_id}: {source_key} -> {output_key}")
|
||||||
|
|
||||||
|
tmp_source = download_to_temp(BUCKET_IN, source_key)
|
||||||
|
ext_out = Path(output_key).suffix or ".mp4"
|
||||||
|
fd, tmp_output = tempfile.mkstemp(suffix=ext_out)
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if preset:
|
||||||
|
config = TranscodeConfig(
|
||||||
|
input_path=tmp_source,
|
||||||
|
output_path=tmp_output,
|
||||||
|
video_codec=preset.get("video_codec", "libx264"),
|
||||||
|
video_bitrate=preset.get("video_bitrate"),
|
||||||
|
video_crf=preset.get("video_crf"),
|
||||||
|
video_preset=preset.get("video_preset"),
|
||||||
|
resolution=preset.get("resolution"),
|
||||||
|
framerate=preset.get("framerate"),
|
||||||
|
audio_codec=preset.get("audio_codec", "aac"),
|
||||||
|
audio_bitrate=preset.get("audio_bitrate"),
|
||||||
|
audio_channels=preset.get("audio_channels"),
|
||||||
|
audio_samplerate=preset.get("audio_samplerate"),
|
||||||
|
container=preset.get("container", "mp4"),
|
||||||
|
extra_args=preset.get("extra_args", []),
|
||||||
|
trim_start=trim_start,
|
||||||
|
trim_end=trim_end,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
config = TranscodeConfig(
|
||||||
|
input_path=tmp_source,
|
||||||
|
output_path=tmp_output,
|
||||||
|
video_codec="copy",
|
||||||
|
audio_codec="copy",
|
||||||
|
trim_start=trim_start,
|
||||||
|
trim_end=trim_end,
|
||||||
|
)
|
||||||
|
|
||||||
|
success = transcode(config, duration=duration)
|
||||||
|
if not success:
|
||||||
|
raise RuntimeError("Transcode returned False")
|
||||||
|
|
||||||
|
logger.info(f"Uploading to {BUCKET_OUT}/{output_key}")
|
||||||
|
upload_file(tmp_output, BUCKET_OUT, output_key)
|
||||||
|
|
||||||
|
_callback(callback_url, job_id, api_key, {"status": "completed"})
|
||||||
|
logger.info(f"Job {job_id} completed")
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception(f"Job {job_id} failed: {e}")
|
||||||
|
_callback(callback_url, job_id, api_key, {"status": "failed", "error": str(e)})
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
for f in [tmp_source, tmp_output]:
|
||||||
|
try:
|
||||||
|
os.unlink(f)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _callback(callback_url: str, job_id: str, api_key: str, payload: dict) -> None:
|
||||||
|
if not callback_url:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
url = f"{callback_url}/jobs/{job_id}/callback"
|
||||||
|
headers = {"X-API-Key": api_key} if api_key else {}
|
||||||
|
resp = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||||
|
logger.info(f"Callback response: {resp.status_code}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Callback failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
148
core/task/lambda_handler.py
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
"""
|
||||||
|
AWS Lambda handler for media transcoding.
|
||||||
|
|
||||||
|
Receives a job payload from Step Functions, downloads source from S3,
|
||||||
|
runs FFmpeg, uploads result to S3, and calls back to the API.
|
||||||
|
|
||||||
|
Uses the same core/ffmpeg module as the local Celery worker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
import requests
|
||||||
|
|
||||||
|
logger = logging.getLogger()
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
# S3 config
|
||||||
|
S3_BUCKET_IN = os.environ.get("S3_BUCKET_IN", "mpr-media-in")
|
||||||
|
S3_BUCKET_OUT = os.environ.get("S3_BUCKET_OUT", "mpr-media-out")
|
||||||
|
AWS_REGION = os.environ.get("AWS_REGION", "us-east-1")
|
||||||
|
|
||||||
|
s3 = boto3.client("s3", region_name=AWS_REGION)
|
||||||
|
|
||||||
|
|
||||||
|
def handler(event, context):
|
||||||
|
"""
|
||||||
|
Lambda entry point.
|
||||||
|
|
||||||
|
Event payload (from Step Functions):
|
||||||
|
{
|
||||||
|
"job_id": "uuid",
|
||||||
|
"source_key": "path/to/source.mp4",
|
||||||
|
"output_key": "output_filename.mp4",
|
||||||
|
"preset": {...} or null,
|
||||||
|
"trim_start": float or null,
|
||||||
|
"trim_end": float or null,
|
||||||
|
"duration": float or null,
|
||||||
|
"callback_url": "https://mpr.mcrn.ar/api",
|
||||||
|
"api_key": "secret"
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
job_id = event["job_id"]
|
||||||
|
source_key = event["source_key"]
|
||||||
|
output_key = event["output_key"]
|
||||||
|
preset = event.get("preset")
|
||||||
|
trim_start = event.get("trim_start")
|
||||||
|
trim_end = event.get("trim_end")
|
||||||
|
duration = event.get("duration")
|
||||||
|
callback_url = event.get("callback_url", "")
|
||||||
|
api_key = event.get("api_key", "")
|
||||||
|
|
||||||
|
logger.info(f"Starting job {job_id}: {source_key} -> {output_key}")
|
||||||
|
|
||||||
|
# Download source from S3
|
||||||
|
ext_in = Path(source_key).suffix or ".mp4"
|
||||||
|
tmp_source = tempfile.mktemp(suffix=ext_in, dir="/tmp")
|
||||||
|
logger.info(f"Downloading s3://{S3_BUCKET_IN}/{source_key}")
|
||||||
|
s3.download_file(S3_BUCKET_IN, source_key, tmp_source)
|
||||||
|
|
||||||
|
# Prepare output temp file
|
||||||
|
ext_out = Path(output_key).suffix or ".mp4"
|
||||||
|
tmp_output = tempfile.mktemp(suffix=ext_out, dir="/tmp")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Import ffmpeg module (bundled in container)
|
||||||
|
from core.ffmpeg.transcode import TranscodeConfig, transcode
|
||||||
|
|
||||||
|
if preset:
|
||||||
|
config = TranscodeConfig(
|
||||||
|
input_path=tmp_source,
|
||||||
|
output_path=tmp_output,
|
||||||
|
video_codec=preset.get("video_codec", "libx264"),
|
||||||
|
video_bitrate=preset.get("video_bitrate"),
|
||||||
|
video_crf=preset.get("video_crf"),
|
||||||
|
video_preset=preset.get("video_preset"),
|
||||||
|
resolution=preset.get("resolution"),
|
||||||
|
framerate=preset.get("framerate"),
|
||||||
|
audio_codec=preset.get("audio_codec", "aac"),
|
||||||
|
audio_bitrate=preset.get("audio_bitrate"),
|
||||||
|
audio_channels=preset.get("audio_channels"),
|
||||||
|
audio_samplerate=preset.get("audio_samplerate"),
|
||||||
|
container=preset.get("container", "mp4"),
|
||||||
|
extra_args=preset.get("extra_args", []),
|
||||||
|
trim_start=trim_start,
|
||||||
|
trim_end=trim_end,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
config = TranscodeConfig(
|
||||||
|
input_path=tmp_source,
|
||||||
|
output_path=tmp_output,
|
||||||
|
video_codec="copy",
|
||||||
|
audio_codec="copy",
|
||||||
|
trim_start=trim_start,
|
||||||
|
trim_end=trim_end,
|
||||||
|
)
|
||||||
|
|
||||||
|
success = transcode(config, duration=duration)
|
||||||
|
|
||||||
|
if not success:
|
||||||
|
raise RuntimeError("Transcode returned False")
|
||||||
|
|
||||||
|
# Upload result to S3
|
||||||
|
logger.info(f"Uploading s3://{S3_BUCKET_OUT}/{output_key}")
|
||||||
|
s3.upload_file(tmp_output, S3_BUCKET_OUT, output_key)
|
||||||
|
|
||||||
|
result = {"status": "completed", "job_id": job_id, "output_key": output_key}
|
||||||
|
|
||||||
|
# Callback to API
|
||||||
|
_callback(callback_url, job_id, api_key, {"status": "completed"})
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception(f"Job {job_id} failed: {e}")
|
||||||
|
|
||||||
|
_callback(callback_url, job_id, api_key, {
|
||||||
|
"status": "failed",
|
||||||
|
"error": str(e),
|
||||||
|
})
|
||||||
|
|
||||||
|
return {"status": "failed", "job_id": job_id, "error": str(e)}
|
||||||
|
|
||||||
|
finally:
|
||||||
|
for f in [tmp_source, tmp_output]:
|
||||||
|
try:
|
||||||
|
os.unlink(f)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _callback(callback_url, job_id, api_key, payload):
|
||||||
|
"""Call back to API with job result."""
|
||||||
|
if not callback_url:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
url = f"{callback_url}/jobs/{job_id}/callback"
|
||||||
|
headers = {}
|
||||||
|
if api_key:
|
||||||
|
headers["X-API-Key"] = api_key
|
||||||
|
resp = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||||
|
logger.info(f"Callback response: {resp.status_code}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Callback failed: {e}")
|
||||||
@@ -8,21 +8,19 @@ from typing import Any, Dict, Optional
|
|||||||
|
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
|
|
||||||
from rpc.server import update_job_progress
|
from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file
|
||||||
from task.executor import get_executor
|
from core.rpc.server import update_job_progress
|
||||||
|
from core.task.executor import get_executor
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Media paths from environment
|
|
||||||
MEDIA_ROOT = os.environ.get("MEDIA_ROOT", "/app/media")
|
|
||||||
|
|
||||||
|
@shared_task(bind=True, queue="transcode", max_retries=3, default_retry_delay=60)
|
||||||
@shared_task(bind=True, max_retries=3, default_retry_delay=60)
|
|
||||||
def run_transcode_job(
|
def run_transcode_job(
|
||||||
self,
|
self,
|
||||||
job_id: str,
|
job_id: str,
|
||||||
source_path: str,
|
source_key: str,
|
||||||
output_path: str,
|
output_key: str,
|
||||||
preset: Optional[Dict[str, Any]] = None,
|
preset: Optional[Dict[str, Any]] = None,
|
||||||
trim_start: Optional[float] = None,
|
trim_start: Optional[float] = None,
|
||||||
trim_end: Optional[float] = None,
|
trim_end: Optional[float] = None,
|
||||||
@@ -31,25 +29,25 @@ def run_transcode_job(
|
|||||||
"""
|
"""
|
||||||
Celery task to run a transcode/trim job.
|
Celery task to run a transcode/trim job.
|
||||||
|
|
||||||
Args:
|
Downloads source from S3, runs FFmpeg, uploads result to S3.
|
||||||
job_id: Unique job identifier
|
|
||||||
source_path: Path to source file
|
|
||||||
output_path: Path for output file
|
|
||||||
preset: Transcode preset dict (optional)
|
|
||||||
trim_start: Trim start time in seconds (optional)
|
|
||||||
trim_end: Trim end time in seconds (optional)
|
|
||||||
duration: Source duration for progress calculation
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Result dict with status and output_path
|
|
||||||
"""
|
"""
|
||||||
logger.info(f"Starting job {job_id}: {source_path} -> {output_path}")
|
logger.info(f"Starting job {job_id}: {source_key} -> {output_key}")
|
||||||
|
|
||||||
# Update status to processing
|
|
||||||
update_job_progress(job_id, progress=0, status="processing")
|
update_job_progress(job_id, progress=0, status="processing")
|
||||||
|
|
||||||
|
# Download source from S3 to temp file
|
||||||
|
logger.info(f"Downloading {source_key} from {BUCKET_IN}")
|
||||||
|
tmp_source = download_to_temp(BUCKET_IN, source_key)
|
||||||
|
|
||||||
|
# Create temp output path with same extension
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ext = Path(output_key).suffix or ".mp4"
|
||||||
|
fd, tmp_output = tempfile.mkstemp(suffix=ext)
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
def progress_callback(percent: int, details: Dict[str, Any]) -> None:
|
def progress_callback(percent: int, details: Dict[str, Any]) -> None:
|
||||||
"""Update gRPC progress state."""
|
|
||||||
update_job_progress(
|
update_job_progress(
|
||||||
job_id,
|
job_id,
|
||||||
progress=percent,
|
progress=percent,
|
||||||
@@ -61,8 +59,8 @@ def run_transcode_job(
|
|||||||
executor = get_executor()
|
executor = get_executor()
|
||||||
success = executor.run(
|
success = executor.run(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
source_path=source_path,
|
source_path=tmp_source,
|
||||||
output_path=output_path,
|
output_path=tmp_output,
|
||||||
preset=preset,
|
preset=preset,
|
||||||
trim_start=trim_start,
|
trim_start=trim_start,
|
||||||
trim_end=trim_end,
|
trim_end=trim_end,
|
||||||
@@ -71,12 +69,16 @@ def run_transcode_job(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if success:
|
if success:
|
||||||
|
# Upload result to S3
|
||||||
|
logger.info(f"Uploading {output_key} to {BUCKET_OUT}")
|
||||||
|
upload_file(tmp_output, BUCKET_OUT, output_key)
|
||||||
|
|
||||||
logger.info(f"Job {job_id} completed successfully")
|
logger.info(f"Job {job_id} completed successfully")
|
||||||
update_job_progress(job_id, progress=100, status="completed")
|
update_job_progress(job_id, progress=100, status="completed")
|
||||||
return {
|
return {
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"job_id": job_id,
|
"job_id": job_id,
|
||||||
"output_path": output_path,
|
"output_key": output_key,
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
raise RuntimeError("Executor returned False")
|
raise RuntimeError("Executor returned False")
|
||||||
@@ -85,7 +87,6 @@ def run_transcode_job(
|
|||||||
logger.exception(f"Job {job_id} failed: {e}")
|
logger.exception(f"Job {job_id} failed: {e}")
|
||||||
update_job_progress(job_id, progress=0, status="failed", error=str(e))
|
update_job_progress(job_id, progress=0, status="failed", error=str(e))
|
||||||
|
|
||||||
# Retry on transient errors
|
|
||||||
if self.request.retries < self.max_retries:
|
if self.request.retries < self.max_retries:
|
||||||
raise self.retry(exc=e)
|
raise self.retry(exc=e)
|
||||||
|
|
||||||
@@ -94,3 +95,11 @@ def run_transcode_job(
|
|||||||
"job_id": job_id,
|
"job_id": job_id,
|
||||||
"error": str(e),
|
"error": str(e),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Clean up temp files
|
||||||
|
for f in [tmp_source, tmp_output]:
|
||||||
|
try:
|
||||||
|
os.unlink(f)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
@@ -16,7 +16,7 @@ REDIS_URL=redis://redis:6379/0
|
|||||||
|
|
||||||
# Django
|
# Django
|
||||||
DEBUG=1
|
DEBUG=1
|
||||||
DJANGO_SETTINGS_MODULE=mpr.settings
|
DJANGO_SETTINGS_MODULE=admin.mpr.settings
|
||||||
SECRET_KEY=change-this-in-production
|
SECRET_KEY=change-this-in-production
|
||||||
|
|
||||||
# Worker
|
# Worker
|
||||||
@@ -27,5 +27,13 @@ GRPC_HOST=grpc
|
|||||||
GRPC_PORT=50051
|
GRPC_PORT=50051
|
||||||
GRPC_MAX_WORKERS=10
|
GRPC_MAX_WORKERS=10
|
||||||
|
|
||||||
|
# S3 Storage (MinIO locally, real S3 on AWS)
|
||||||
|
S3_ENDPOINT_URL=http://minio:9000
|
||||||
|
S3_BUCKET_IN=mpr-media-in
|
||||||
|
S3_BUCKET_OUT=mpr-media-out
|
||||||
|
AWS_REGION=us-east-1
|
||||||
|
AWS_ACCESS_KEY_ID=minioadmin
|
||||||
|
AWS_SECRET_ACCESS_KEY=minioadmin
|
||||||
|
|
||||||
# Vite
|
# Vite
|
||||||
VITE_ALLOWED_HOSTS=your-domain.local
|
VITE_ALLOWED_HOSTS=your-domain.local
|
||||||
|
|||||||
@@ -1,14 +1,10 @@
|
|||||||
FROM python:3.11-slim
|
FROM python:3.11-slim
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
ffmpeg \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
COPY . .
|
# No COPY . . — code is volume-mounted in dev (..:/app)
|
||||||
|
|
||||||
CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
|
CMD ["python", "admin/manage.py", "runserver", "0.0.0.0:8000"]
|
||||||
|
|||||||
14
ctrl/Dockerfile.worker
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
ffmpeg \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY requirements.txt requirements-worker.txt ./
|
||||||
|
RUN pip install --no-cache-dir -r requirements-worker.txt
|
||||||
|
|
||||||
|
# No COPY . . — code is volume-mounted in dev (..:/app)
|
||||||
|
|
||||||
|
CMD ["celery", "-A", "admin.mpr", "worker", "--loglevel=info"]
|
||||||
317
ctrl/deploy.sh
@@ -1,18 +1,17 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Deploy MPR to remote server via rsync
|
# MPR Deploy Script
|
||||||
# Uses project .gitignore for excludes
|
|
||||||
#
|
#
|
||||||
# Usage: ./ctrl/deploy.sh [--restart] [--dry-run]
|
# Usage: ./ctrl/deploy.sh <command> [options]
|
||||||
#
|
#
|
||||||
# Examples:
|
# Commands:
|
||||||
# ./ctrl/deploy.sh # Sync files only
|
# rsync [--restart] [--dry-run] Sync to remote server via rsync
|
||||||
# ./ctrl/deploy.sh --restart # Sync and restart services
|
# aws Deploy AWS infrastructure (Lambda, Step Functions, S3)
|
||||||
# ./ctrl/deploy.sh --dry-run # Preview sync
|
|
||||||
|
|
||||||
set -e
|
set -euo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
|
||||||
source "$SCRIPT_DIR/.env" 2>/dev/null || true
|
source "$SCRIPT_DIR/.env" 2>/dev/null || true
|
||||||
|
|
||||||
@@ -21,56 +20,268 @@ GREEN='\033[0;32m'
|
|||||||
YELLOW='\033[1;33m'
|
YELLOW='\033[1;33m'
|
||||||
NC='\033[0m'
|
NC='\033[0m'
|
||||||
|
|
||||||
if [ -z "$SERVER" ] || [ -z "$REMOTE_PATH" ]; then
|
# ─── Rsync Deploy ─────────────────────────────────────────────────────────────
|
||||||
echo -e "${RED}Error: SERVER and REMOTE_PATH must be set in ctrl/.env${NC}"
|
|
||||||
echo "Example:"
|
|
||||||
echo " SERVER=user@host"
|
|
||||||
echo " REMOTE_PATH=~/mpr"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
RESTART=false
|
deploy_rsync() {
|
||||||
DRY_RUN=""
|
if [ -z "${SERVER:-}" ] || [ -z "${REMOTE_PATH:-}" ]; then
|
||||||
|
echo -e "${RED}Error: SERVER and REMOTE_PATH must be set in ctrl/.env${NC}"
|
||||||
|
echo "Example:"
|
||||||
|
echo " SERVER=user@host"
|
||||||
|
echo " REMOTE_PATH=~/mpr"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
while [ $# -gt 0 ]; do
|
RESTART=false
|
||||||
case "$1" in
|
DRY_RUN=""
|
||||||
--restart)
|
|
||||||
RESTART=true
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--dry-run)
|
|
||||||
DRY_RUN="--dry-run"
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown option: $1"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
echo -e "${GREEN}=== Deploying MPR to $SERVER:$REMOTE_PATH ===${NC}"
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--restart) RESTART=true; shift ;;
|
||||||
|
--dry-run) DRY_RUN="--dry-run"; shift ;;
|
||||||
|
*) echo "Unknown option: $1"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# Sync files using .gitignore for excludes
|
echo -e "${GREEN}=== Deploying MPR to $SERVER:$REMOTE_PATH ===${NC}"
|
||||||
echo -e "${YELLOW}Syncing files...${NC}"
|
|
||||||
rsync -avz --delete $DRY_RUN \
|
|
||||||
--filter=':- .gitignore' \
|
|
||||||
--exclude='.git' \
|
|
||||||
--exclude='media/*' \
|
|
||||||
--exclude='ctrl/.env' \
|
|
||||||
"$PROJECT_ROOT/" "$SERVER:$REMOTE_PATH/"
|
|
||||||
|
|
||||||
if [ -n "$DRY_RUN" ]; then
|
echo -e "${YELLOW}Syncing files...${NC}"
|
||||||
echo -e "${YELLOW}Dry run - no changes made${NC}"
|
rsync -avz --delete $DRY_RUN \
|
||||||
exit 0
|
--filter=':- .gitignore' \
|
||||||
fi
|
--exclude='.git' \
|
||||||
|
--exclude='media/*' \
|
||||||
|
--exclude='ctrl/.env' \
|
||||||
|
"$PROJECT_ROOT/" "$SERVER:$REMOTE_PATH/"
|
||||||
|
|
||||||
# Copy env template if .env doesn't exist on remote
|
if [ -n "$DRY_RUN" ]; then
|
||||||
ssh "$SERVER" "[ -f $REMOTE_PATH/ctrl/.env ] || cp $REMOTE_PATH/ctrl/.env.template $REMOTE_PATH/ctrl/.env"
|
echo -e "${YELLOW}Dry run - no changes made${NC}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
if [ "$RESTART" = true ]; then
|
ssh "$SERVER" "[ -f $REMOTE_PATH/ctrl/.env ] || cp $REMOTE_PATH/ctrl/.env.template $REMOTE_PATH/ctrl/.env"
|
||||||
echo -e "${YELLOW}Restarting services...${NC}"
|
|
||||||
ssh "$SERVER" "cd $REMOTE_PATH/ctrl && docker compose down && docker compose up -d --build"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo -e "${GREEN}Done!${NC}"
|
if [ "$RESTART" = true ]; then
|
||||||
|
echo -e "${YELLOW}Restarting services...${NC}"
|
||||||
|
ssh "$SERVER" "cd $REMOTE_PATH/ctrl && docker compose down && docker compose up -d --build"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "${GREEN}Done!${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── AWS Deploy ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
deploy_aws() {
|
||||||
|
REGION="${AWS_REGION:-us-east-1}"
|
||||||
|
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
|
||||||
|
PROJECT="mpr"
|
||||||
|
|
||||||
|
# S3
|
||||||
|
BUCKET_IN="${S3_BUCKET_IN:-mpr-media-in}"
|
||||||
|
BUCKET_OUT="${S3_BUCKET_OUT:-mpr-media-out}"
|
||||||
|
|
||||||
|
# ECR
|
||||||
|
ECR_REPO="${PROJECT}-transcode"
|
||||||
|
ECR_URI="${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${ECR_REPO}"
|
||||||
|
|
||||||
|
# Lambda
|
||||||
|
LAMBDA_NAME="${PROJECT}-transcode"
|
||||||
|
LAMBDA_TIMEOUT=900
|
||||||
|
LAMBDA_MEMORY=2048
|
||||||
|
|
||||||
|
# Step Functions
|
||||||
|
SFN_NAME="${PROJECT}-transcode"
|
||||||
|
|
||||||
|
# IAM
|
||||||
|
LAMBDA_ROLE_NAME="${PROJECT}-lambda-role"
|
||||||
|
SFN_ROLE_NAME="${PROJECT}-sfn-role"
|
||||||
|
|
||||||
|
# Callback
|
||||||
|
CALLBACK_URL="${CALLBACK_URL:-https://mpr.mcrn.ar/api}"
|
||||||
|
CALLBACK_API_KEY="${CALLBACK_API_KEY:-changeme}"
|
||||||
|
|
||||||
|
echo -e "${GREEN}=== Deploying MPR to AWS ($REGION, account $ACCOUNT_ID) ===${NC}"
|
||||||
|
|
||||||
|
# ─── S3 Buckets ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Creating S3 buckets...${NC}"
|
||||||
|
for bucket in "$BUCKET_IN" "$BUCKET_OUT"; do
|
||||||
|
if ! aws s3api head-bucket --bucket "$bucket" 2>/dev/null; then
|
||||||
|
aws s3api create-bucket \
|
||||||
|
--bucket "$bucket" \
|
||||||
|
--region "$REGION" \
|
||||||
|
--create-bucket-configuration LocationConstraint="$REGION"
|
||||||
|
echo " Created $bucket"
|
||||||
|
else
|
||||||
|
echo " $bucket already exists"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# ─── IAM Roles ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Creating IAM roles...${NC}"
|
||||||
|
|
||||||
|
if ! aws iam get-role --role-name "$LAMBDA_ROLE_NAME" 2>/dev/null; then
|
||||||
|
aws iam create-role \
|
||||||
|
--role-name "$LAMBDA_ROLE_NAME" \
|
||||||
|
--assume-role-policy-document '{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Principal": {"Service": "lambda.amazonaws.com"},
|
||||||
|
"Action": "sts:AssumeRole"
|
||||||
|
}]
|
||||||
|
}'
|
||||||
|
aws iam attach-role-policy \
|
||||||
|
--role-name "$LAMBDA_ROLE_NAME" \
|
||||||
|
--policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
|
||||||
|
aws iam put-role-policy \
|
||||||
|
--role-name "$LAMBDA_ROLE_NAME" \
|
||||||
|
--policy-name "${PROJECT}-s3-access" \
|
||||||
|
--policy-document '{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:PutObject"],
|
||||||
|
"Resource": [
|
||||||
|
"arn:aws:s3:::'"$BUCKET_IN"'/*",
|
||||||
|
"arn:aws:s3:::'"$BUCKET_OUT"'/*"
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
}'
|
||||||
|
echo " Created $LAMBDA_ROLE_NAME"
|
||||||
|
echo " Waiting for role to propagate..."
|
||||||
|
sleep 10
|
||||||
|
else
|
||||||
|
echo " $LAMBDA_ROLE_NAME already exists"
|
||||||
|
fi
|
||||||
|
LAMBDA_ROLE_ARN=$(aws iam get-role --role-name "$LAMBDA_ROLE_NAME" --query Role.Arn --output text)
|
||||||
|
|
||||||
|
if ! aws iam get-role --role-name "$SFN_ROLE_NAME" 2>/dev/null; then
|
||||||
|
aws iam create-role \
|
||||||
|
--role-name "$SFN_ROLE_NAME" \
|
||||||
|
--assume-role-policy-document '{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Principal": {"Service": "states.amazonaws.com"},
|
||||||
|
"Action": "sts:AssumeRole"
|
||||||
|
}]
|
||||||
|
}'
|
||||||
|
aws iam put-role-policy \
|
||||||
|
--role-name "$SFN_ROLE_NAME" \
|
||||||
|
--policy-name "${PROJECT}-sfn-invoke-lambda" \
|
||||||
|
--policy-document '{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": "lambda:InvokeFunction",
|
||||||
|
"Resource": "arn:aws:lambda:'"$REGION"':'"$ACCOUNT_ID"':function:'"$LAMBDA_NAME"'"
|
||||||
|
}]
|
||||||
|
}'
|
||||||
|
echo " Created $SFN_ROLE_NAME"
|
||||||
|
sleep 10
|
||||||
|
else
|
||||||
|
echo " $SFN_ROLE_NAME already exists"
|
||||||
|
fi
|
||||||
|
SFN_ROLE_ARN=$(aws iam get-role --role-name "$SFN_ROLE_NAME" --query Role.Arn --output text)
|
||||||
|
|
||||||
|
# ─── ECR Repository ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Setting up ECR...${NC}"
|
||||||
|
if ! aws ecr describe-repositories --repository-names "$ECR_REPO" --region "$REGION" 2>/dev/null; then
|
||||||
|
aws ecr create-repository --repository-name "$ECR_REPO" --region "$REGION"
|
||||||
|
echo " Created ECR repo $ECR_REPO"
|
||||||
|
else
|
||||||
|
echo " ECR repo $ECR_REPO already exists"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ─── Build & Push Lambda Image ───────────────────────────────────────
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Building Lambda container image...${NC}"
|
||||||
|
|
||||||
|
docker build -f ctrl/lambda/Dockerfile -t "${ECR_REPO}:latest" .
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Pushing to ECR...${NC}"
|
||||||
|
aws ecr get-login-password --region "$REGION" | \
|
||||||
|
docker login --username AWS --password-stdin "${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com"
|
||||||
|
|
||||||
|
docker tag "${ECR_REPO}:latest" "${ECR_URI}:latest"
|
||||||
|
docker push "${ECR_URI}:latest"
|
||||||
|
|
||||||
|
# ─── Lambda Function ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Deploying Lambda function...${NC}"
|
||||||
|
LAMBDA_ARN="arn:aws:lambda:${REGION}:${ACCOUNT_ID}:function:${LAMBDA_NAME}"
|
||||||
|
|
||||||
|
if aws lambda get-function --function-name "$LAMBDA_NAME" --region "$REGION" 2>/dev/null; then
|
||||||
|
aws lambda update-function-code \
|
||||||
|
--function-name "$LAMBDA_NAME" \
|
||||||
|
--image-uri "${ECR_URI}:latest" \
|
||||||
|
--region "$REGION"
|
||||||
|
echo " Updated $LAMBDA_NAME"
|
||||||
|
else
|
||||||
|
aws lambda create-function \
|
||||||
|
--function-name "$LAMBDA_NAME" \
|
||||||
|
--package-type Image \
|
||||||
|
--code ImageUri="${ECR_URI}:latest" \
|
||||||
|
--role "$LAMBDA_ROLE_ARN" \
|
||||||
|
--timeout "$LAMBDA_TIMEOUT" \
|
||||||
|
--memory-size "$LAMBDA_MEMORY" \
|
||||||
|
--environment "Variables={S3_BUCKET_IN=${BUCKET_IN},S3_BUCKET_OUT=${BUCKET_OUT},AWS_REGION=${REGION}}" \
|
||||||
|
--region "$REGION"
|
||||||
|
echo " Created $LAMBDA_NAME"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ─── Step Functions ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Deploying Step Functions state machine...${NC}"
|
||||||
|
|
||||||
|
SFN_DEFINITION=$(sed "s|\${TranscodeLambdaArn}|${LAMBDA_ARN}|g" ctrl/state-machine.json)
|
||||||
|
|
||||||
|
SFN_ARN="arn:aws:states:${REGION}:${ACCOUNT_ID}:stateMachine:${SFN_NAME}"
|
||||||
|
if aws stepfunctions describe-state-machine --state-machine-arn "$SFN_ARN" --region "$REGION" 2>/dev/null; then
|
||||||
|
aws stepfunctions update-state-machine \
|
||||||
|
--state-machine-arn "$SFN_ARN" \
|
||||||
|
--definition "$SFN_DEFINITION" \
|
||||||
|
--region "$REGION"
|
||||||
|
echo " Updated $SFN_NAME"
|
||||||
|
else
|
||||||
|
aws stepfunctions create-state-machine \
|
||||||
|
--name "$SFN_NAME" \
|
||||||
|
--definition "$SFN_DEFINITION" \
|
||||||
|
--role-arn "$SFN_ROLE_ARN" \
|
||||||
|
--region "$REGION"
|
||||||
|
echo " Created $SFN_NAME"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ─── Summary ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo -e "${GREEN}Deployment complete!${NC}"
|
||||||
|
echo ""
|
||||||
|
echo "Add these to your .env:"
|
||||||
|
echo " MPR_EXECUTOR=lambda"
|
||||||
|
echo " STEP_FUNCTION_ARN=${SFN_ARN}"
|
||||||
|
echo " LAMBDA_FUNCTION_ARN=${LAMBDA_ARN}"
|
||||||
|
echo " S3_BUCKET_IN=${BUCKET_IN}"
|
||||||
|
echo " S3_BUCKET_OUT=${BUCKET_OUT}"
|
||||||
|
echo " CALLBACK_URL=${CALLBACK_URL}"
|
||||||
|
echo " CALLBACK_API_KEY=${CALLBACK_API_KEY}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── Main ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
COMMAND="${1:-}"
|
||||||
|
shift || true
|
||||||
|
|
||||||
|
case "$COMMAND" in
|
||||||
|
rsync) deploy_rsync "$@" ;;
|
||||||
|
aws) deploy_aws "$@" ;;
|
||||||
|
*)
|
||||||
|
echo "Usage: ./ctrl/deploy.sh <command> [options]"
|
||||||
|
echo ""
|
||||||
|
echo "Commands:"
|
||||||
|
echo " rsync [--restart] [--dry-run] Sync to remote server"
|
||||||
|
echo " aws Deploy AWS infrastructure"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|||||||
@@ -1,10 +1,16 @@
|
|||||||
x-common-env: &common-env
|
x-common-env: &common-env
|
||||||
DATABASE_URL: postgresql://mpr_user:mpr_pass@postgres:5432/mpr
|
DATABASE_URL: postgresql://mpr_user:mpr_pass@postgres:5432/mpr
|
||||||
REDIS_URL: redis://redis:6379/0
|
REDIS_URL: redis://redis:6379/0
|
||||||
DJANGO_SETTINGS_MODULE: mpr.settings
|
DJANGO_SETTINGS_MODULE: admin.mpr.settings
|
||||||
DEBUG: 1
|
DEBUG: 1
|
||||||
GRPC_HOST: grpc
|
GRPC_HOST: grpc
|
||||||
GRPC_PORT: 50051
|
GRPC_PORT: 50051
|
||||||
|
S3_ENDPOINT_URL: http://minio:9000
|
||||||
|
S3_BUCKET_IN: mpr-media-in
|
||||||
|
S3_BUCKET_OUT: mpr-media-out
|
||||||
|
AWS_ACCESS_KEY_ID: minioadmin
|
||||||
|
AWS_SECRET_ACCESS_KEY: minioadmin
|
||||||
|
AWS_REGION: us-east-1
|
||||||
|
|
||||||
x-healthcheck-defaults: &healthcheck-defaults
|
x-healthcheck-defaults: &healthcheck-defaults
|
||||||
interval: 5s
|
interval: 5s
|
||||||
@@ -40,17 +46,46 @@ services:
|
|||||||
<<: *healthcheck-defaults
|
<<: *healthcheck-defaults
|
||||||
test: ["CMD", "redis-cli", "ping"]
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
|
||||||
|
minio:
|
||||||
|
image: minio/minio
|
||||||
|
command: ["server", "/data", "--console-address", ":9001"]
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minioadmin
|
||||||
|
MINIO_ROOT_PASSWORD: minioadmin
|
||||||
|
volumes:
|
||||||
|
- minio-data:/data
|
||||||
|
healthcheck:
|
||||||
|
<<: *healthcheck-defaults
|
||||||
|
test: ["CMD", "mc", "ready", "local"]
|
||||||
|
|
||||||
|
minio-init:
|
||||||
|
image: minio/mc
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
entrypoint: ["/bin/sh", "-c"]
|
||||||
|
command:
|
||||||
|
- |
|
||||||
|
mc alias set local http://minio:9000 minioadmin minioadmin
|
||||||
|
mc mb --ignore-existing local/mpr-media-in
|
||||||
|
mc mb --ignore-existing local/mpr-media-out
|
||||||
|
mc anonymous set download local/mpr-media-in
|
||||||
|
mc anonymous set download local/mpr-media-out
|
||||||
|
|
||||||
nginx:
|
nginx:
|
||||||
image: nginx:alpine
|
image: nginx:alpine
|
||||||
ports:
|
ports:
|
||||||
- "80:80"
|
- "80:80"
|
||||||
volumes:
|
volumes:
|
||||||
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
- ../media:/app/media:ro
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- django
|
- django
|
||||||
- fastapi
|
- fastapi
|
||||||
- timeline
|
- timeline
|
||||||
|
- minio
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Application Services
|
# Application Services
|
||||||
@@ -61,16 +96,15 @@ services:
|
|||||||
context: ..
|
context: ..
|
||||||
dockerfile: ctrl/Dockerfile
|
dockerfile: ctrl/Dockerfile
|
||||||
command: >
|
command: >
|
||||||
bash -c "python manage.py migrate &&
|
bash -c "python admin/manage.py migrate &&
|
||||||
python manage.py loadbuiltins || true &&
|
python admin/manage.py loadbuiltins || true &&
|
||||||
python manage.py runserver 0.0.0.0:8701"
|
python admin/manage.py runserver 0.0.0.0:8701"
|
||||||
ports:
|
ports:
|
||||||
- "8701:8701"
|
- "8701:8701"
|
||||||
environment:
|
environment:
|
||||||
<<: *common-env
|
<<: *common-env
|
||||||
volumes:
|
volumes:
|
||||||
- ..:/app
|
- ..:/app
|
||||||
- ../media:/app/media
|
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -81,14 +115,14 @@ services:
|
|||||||
build:
|
build:
|
||||||
context: ..
|
context: ..
|
||||||
dockerfile: ctrl/Dockerfile
|
dockerfile: ctrl/Dockerfile
|
||||||
command: uvicorn api.main:app --host 0.0.0.0 --port 8702 --reload
|
command: uvicorn core.api.main:app --host 0.0.0.0 --port 8702 --reload
|
||||||
ports:
|
ports:
|
||||||
- "8702:8702"
|
- "8702:8702"
|
||||||
environment:
|
environment:
|
||||||
<<: *common-env
|
<<: *common-env
|
||||||
|
DJANGO_ALLOW_ASYNC_UNSAFE: "true"
|
||||||
volumes:
|
volumes:
|
||||||
- ..:/app
|
- ..:/app
|
||||||
- ../media:/app/media
|
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -99,7 +133,7 @@ services:
|
|||||||
build:
|
build:
|
||||||
context: ..
|
context: ..
|
||||||
dockerfile: ctrl/Dockerfile
|
dockerfile: ctrl/Dockerfile
|
||||||
command: python -m rpc.server
|
command: python -m core.rpc.server
|
||||||
ports:
|
ports:
|
||||||
- "50052:50051"
|
- "50052:50051"
|
||||||
environment:
|
environment:
|
||||||
@@ -108,7 +142,6 @@ services:
|
|||||||
GRPC_MAX_WORKERS: 10
|
GRPC_MAX_WORKERS: 10
|
||||||
volumes:
|
volumes:
|
||||||
- ..:/app
|
- ..:/app
|
||||||
- ../media:/app/media
|
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -118,14 +151,13 @@ services:
|
|||||||
celery:
|
celery:
|
||||||
build:
|
build:
|
||||||
context: ..
|
context: ..
|
||||||
dockerfile: ctrl/Dockerfile
|
dockerfile: ctrl/Dockerfile.worker
|
||||||
command: celery -A mpr worker -l info -Q default -c 2
|
command: celery -A admin.mpr worker -l info -Q transcode -c 2
|
||||||
environment:
|
environment:
|
||||||
<<: *common-env
|
<<: *common-env
|
||||||
MPR_EXECUTOR: local
|
MPR_EXECUTOR: local
|
||||||
volumes:
|
volumes:
|
||||||
- ..:/app
|
- ..:/app
|
||||||
- ../media:/app/media
|
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -148,6 +180,7 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
postgres-data:
|
postgres-data:
|
||||||
redis-data:
|
redis-data:
|
||||||
|
minio-data:
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -1,46 +1,22 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Model generation script for MPR
|
# Model generation script for MPR
|
||||||
# Generates Django, Pydantic, TypeScript, and Protobuf from schema/models
|
# Generates all targets from core/schema/modelgen.json config
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
echo "Generating models from schema/models..."
|
echo "Generating models from core/schema/models..."
|
||||||
|
python -m modelgen generate --config core/schema/modelgen.json
|
||||||
# Django ORM models
|
|
||||||
python -m modelgen from-schema \
|
|
||||||
--schema schema/models \
|
|
||||||
--output mpr/media_assets/models.py \
|
|
||||||
--targets django
|
|
||||||
|
|
||||||
# Pydantic schemas for FastAPI
|
|
||||||
python -m modelgen from-schema \
|
|
||||||
--schema schema/models \
|
|
||||||
--output api/schemas/models.py \
|
|
||||||
--targets pydantic
|
|
||||||
|
|
||||||
# TypeScript types for Timeline UI
|
|
||||||
python -m modelgen from-schema \
|
|
||||||
--schema schema/models \
|
|
||||||
--output ui/timeline/src/types.ts \
|
|
||||||
--targets typescript
|
|
||||||
|
|
||||||
# Protobuf for gRPC
|
|
||||||
python -m modelgen from-schema \
|
|
||||||
--schema schema/models \
|
|
||||||
--output rpc/protos/worker.proto \
|
|
||||||
--targets proto
|
|
||||||
|
|
||||||
# Generate gRPC stubs from proto
|
# Generate gRPC stubs from proto
|
||||||
echo "Generating gRPC stubs..."
|
echo "Generating gRPC stubs..."
|
||||||
python -m grpc_tools.protoc \
|
python -m grpc_tools.protoc \
|
||||||
-I rpc/protos \
|
-I core/rpc/protos \
|
||||||
--python_out=rpc \
|
--python_out=core/rpc \
|
||||||
--grpc_python_out=rpc \
|
--grpc_python_out=core/rpc \
|
||||||
rpc/protos/worker.proto
|
core/rpc/protos/worker.proto
|
||||||
|
|
||||||
# Fix relative import in generated grpc stub
|
# Fix relative import in generated grpc stub
|
||||||
sed -i 's/^import worker_pb2/from . import worker_pb2/' rpc/worker_pb2_grpc.py
|
sed -i 's/^import worker_pb2/from . import worker_pb2/' core/rpc/worker_pb2_grpc.py
|
||||||
|
|
||||||
echo "Done!"
|
echo "Done!"
|
||||||
|
|||||||
21
ctrl/lambda/Dockerfile
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
FROM public.ecr.aws/lambda/python:3.11
|
||||||
|
|
||||||
|
# Install ffmpeg static binary
|
||||||
|
RUN yum install -y tar xz && \
|
||||||
|
curl -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o /tmp/ffmpeg.tar.xz && \
|
||||||
|
tar -xf /tmp/ffmpeg.tar.xz -C /tmp && \
|
||||||
|
cp /tmp/ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ffmpeg && \
|
||||||
|
cp /tmp/ffmpeg-*-amd64-static/ffprobe /usr/local/bin/ffprobe && \
|
||||||
|
rm -rf /tmp/ffmpeg* && \
|
||||||
|
yum clean all
|
||||||
|
|
||||||
|
# Install Python dependencies
|
||||||
|
COPY ctrl/lambda/requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
# Copy application code
|
||||||
|
COPY core/task/lambda_handler.py ${LAMBDA_TASK_ROOT}/core/task/lambda_handler.py
|
||||||
|
COPY core/task/__init__.py ${LAMBDA_TASK_ROOT}/core/task/__init__.py
|
||||||
|
COPY core/ ${LAMBDA_TASK_ROOT}/core/
|
||||||
|
|
||||||
|
CMD ["core.task.lambda_handler.handler"]
|
||||||
2
ctrl/lambda/requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
ffmpeg-python>=0.2.0
|
||||||
|
requests>=2.31.0
|
||||||
@@ -21,6 +21,10 @@ http {
|
|||||||
server timeline:5173;
|
server timeline:5173;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
upstream minio {
|
||||||
|
server minio:9000;
|
||||||
|
}
|
||||||
|
|
||||||
server {
|
server {
|
||||||
listen 80;
|
listen 80;
|
||||||
server_name mpr.local.ar;
|
server_name mpr.local.ar;
|
||||||
@@ -40,9 +44,9 @@ http {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
}
|
}
|
||||||
|
|
||||||
# FastAPI
|
# FastAPI — trailing slash strips /api prefix before forwarding
|
||||||
location /api/ {
|
location /api/ {
|
||||||
proxy_pass http://fastapi;
|
proxy_pass http://fastapi/;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
@@ -67,10 +71,15 @@ http {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
}
|
}
|
||||||
|
|
||||||
# Media files
|
# Media files - proxied from MinIO (local) or S3 (AWS)
|
||||||
location /media {
|
location /media/in/ {
|
||||||
alias /app/media;
|
proxy_pass http://minio/mpr-media-in/;
|
||||||
autoindex on;
|
proxy_set_header Host $http_host;
|
||||||
|
}
|
||||||
|
|
||||||
|
location /media/out/ {
|
||||||
|
proxy_pass http://minio/mpr-media-out/;
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
}
|
}
|
||||||
|
|
||||||
# Default to Timeline UI
|
# Default to Timeline UI
|
||||||
|
|||||||
39
ctrl/state-machine.json
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"Comment": "MPR Transcode Job - orchestrates Lambda-based media transcoding",
|
||||||
|
"StartAt": "Transcode",
|
||||||
|
"States": {
|
||||||
|
"Transcode": {
|
||||||
|
"Type": "Task",
|
||||||
|
"Resource": "${TranscodeLambdaArn}",
|
||||||
|
"TimeoutSeconds": 900,
|
||||||
|
"Retry": [
|
||||||
|
{
|
||||||
|
"ErrorEquals": ["States.TaskFailed", "Lambda.ServiceException"],
|
||||||
|
"IntervalSeconds": 10,
|
||||||
|
"MaxAttempts": 2,
|
||||||
|
"BackoffRate": 2.0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"Catch": [
|
||||||
|
{
|
||||||
|
"ErrorEquals": ["States.ALL"],
|
||||||
|
"Next": "HandleError",
|
||||||
|
"ResultPath": "$.error"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"Next": "Done"
|
||||||
|
},
|
||||||
|
"HandleError": {
|
||||||
|
"Type": "Pass",
|
||||||
|
"Parameters": {
|
||||||
|
"status": "failed",
|
||||||
|
"job_id.$": "$.job_id",
|
||||||
|
"error.$": "$.error.Cause"
|
||||||
|
},
|
||||||
|
"Next": "Done"
|
||||||
|
},
|
||||||
|
"Done": {
|
||||||
|
"Type": "Succeed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
digraph system_overview {
|
|
||||||
rankdir=TB
|
|
||||||
node [shape=box, style=rounded, fontname="Helvetica"]
|
|
||||||
edge [fontname="Helvetica", fontsize=10]
|
|
||||||
|
|
||||||
// Title
|
|
||||||
labelloc="t"
|
|
||||||
label="MPR - System Overview"
|
|
||||||
fontsize=16
|
|
||||||
fontname="Helvetica-Bold"
|
|
||||||
|
|
||||||
// Styling
|
|
||||||
graph [splines=ortho, nodesep=0.8, ranksep=0.8]
|
|
||||||
|
|
||||||
// External
|
|
||||||
subgraph cluster_external {
|
|
||||||
label="External"
|
|
||||||
style=dashed
|
|
||||||
color=gray
|
|
||||||
|
|
||||||
browser [label="Browser\nmpr.local.ar", shape=ellipse]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nginx reverse proxy
|
|
||||||
subgraph cluster_proxy {
|
|
||||||
label="Reverse Proxy"
|
|
||||||
style=filled
|
|
||||||
fillcolor="#e8f4f8"
|
|
||||||
|
|
||||||
nginx [label="nginx\nport 80"]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Application layer
|
|
||||||
subgraph cluster_apps {
|
|
||||||
label="Application Layer"
|
|
||||||
style=filled
|
|
||||||
fillcolor="#f0f8e8"
|
|
||||||
|
|
||||||
django [label="Django\n/admin\nport 8701"]
|
|
||||||
fastapi [label="FastAPI\n/api\nport 8702"]
|
|
||||||
timeline [label="Timeline UI\n/ui\nport 5173"]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Worker layer
|
|
||||||
subgraph cluster_workers {
|
|
||||||
label="Worker Layer"
|
|
||||||
style=filled
|
|
||||||
fillcolor="#fff8e8"
|
|
||||||
|
|
||||||
grpc_server [label="gRPC Server\nport 50051"]
|
|
||||||
celery [label="Celery Worker\n(local)"]
|
|
||||||
lambda [label="Lambda\n(cloud)", style="dashed,rounded"]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Data layer
|
|
||||||
subgraph cluster_data {
|
|
||||||
label="Data Layer"
|
|
||||||
style=filled
|
|
||||||
fillcolor="#f8e8f0"
|
|
||||||
|
|
||||||
postgres [label="PostgreSQL\nport 5433", shape=cylinder]
|
|
||||||
redis [label="Redis\nport 6380", shape=cylinder]
|
|
||||||
sqs [label="SQS\n(cloud)", shape=cylinder, style=dashed]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Storage
|
|
||||||
subgraph cluster_storage {
|
|
||||||
label="File Storage"
|
|
||||||
style=filled
|
|
||||||
fillcolor="#f0f0f0"
|
|
||||||
|
|
||||||
local_fs [label="Local FS\n/media", shape=folder]
|
|
||||||
s3 [label="S3\n(cloud)", shape=folder, style=dashed]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connections
|
|
||||||
browser -> nginx
|
|
||||||
|
|
||||||
nginx -> django [label="/admin"]
|
|
||||||
nginx -> fastapi [label="/api"]
|
|
||||||
nginx -> timeline [label="/ui"]
|
|
||||||
|
|
||||||
// Django uses FastAPI for operations (single API gateway)
|
|
||||||
django -> fastapi [label="job operations"]
|
|
||||||
django -> postgres [label="CRUD only"]
|
|
||||||
|
|
||||||
// Timeline UI uses FastAPI
|
|
||||||
timeline -> fastapi [label="REST API"]
|
|
||||||
|
|
||||||
// FastAPI is the single API gateway
|
|
||||||
fastapi -> postgres
|
|
||||||
fastapi -> redis [label="job status"]
|
|
||||||
fastapi -> grpc_server [label="gRPC\nprogress streaming"]
|
|
||||||
|
|
||||||
// Worker layer
|
|
||||||
grpc_server -> celery [label="task dispatch"]
|
|
||||||
celery -> redis [label="queue"]
|
|
||||||
celery -> postgres [label="job updates"]
|
|
||||||
celery -> grpc_server [label="progress\ncallbacks", style=dotted]
|
|
||||||
celery -> local_fs [label="read/write"]
|
|
||||||
|
|
||||||
// Cloud (future)
|
|
||||||
lambda -> sqs [label="queue", style=dashed]
|
|
||||||
lambda -> s3 [label="read/write", style=dashed]
|
|
||||||
}
|
|
||||||
@@ -1,260 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
||||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
|
||||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
|
||||||
<!-- Generated by graphviz version 14.1.1 (0)
|
|
||||||
-->
|
|
||||||
<!-- Title: system_overview Pages: 1 -->
|
|
||||||
<svg width="843pt" height="957pt"
|
|
||||||
viewBox="0.00 0.00 843.00 957.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
|
||||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 952.79)">
|
|
||||||
<title>system_overview</title>
|
|
||||||
<polygon fill="white" stroke="none" points="-4,4 -4,-952.79 838.5,-952.79 838.5,4 -4,4"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="417.25" y="-929.59" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - System Overview</text>
|
|
||||||
<g id="clust1" class="cluster">
|
|
||||||
<title>cluster_external</title>
|
|
||||||
<polygon fill="none" stroke="gray" stroke-dasharray="5,2" points="478,-809.69 478,-913.29 632,-913.29 632,-809.69 478,-809.69"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-894.09" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">External</text>
|
|
||||||
</g>
|
|
||||||
<g id="clust2" class="cluster">
|
|
||||||
<title>cluster_proxy</title>
|
|
||||||
<polygon fill="#e8f4f8" stroke="black" points="482,-693.69 482,-779.69 628,-779.69 628,-693.69 482,-693.69"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-760.49" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Reverse Proxy</text>
|
|
||||||
</g>
|
|
||||||
<g id="clust3" class="cluster">
|
|
||||||
<title>cluster_apps</title>
|
|
||||||
<polygon fill="#f0f8e8" stroke="black" points="352,-418.19 352,-651.94 606,-651.94 606,-418.19 352,-418.19"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="479" y="-632.74" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Application Layer</text>
|
|
||||||
</g>
|
|
||||||
<g id="clust4" class="cluster">
|
|
||||||
<title>cluster_workers</title>
|
|
||||||
<polygon fill="#fff8e8" stroke="black" points="125,-151.69 125,-363.69 374,-363.69 374,-151.69 125,-151.69"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="249.5" y="-344.49" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Worker Layer</text>
|
|
||||||
</g>
|
|
||||||
<g id="clust5" class="cluster">
|
|
||||||
<title>cluster_data</title>
|
|
||||||
<polygon fill="#f8e8f0" stroke="black" points="322,-8 322,-109.94 700,-109.94 700,-8 322,-8"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="511" y="-90.74" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Data Layer</text>
|
|
||||||
</g>
|
|
||||||
<g id="clust6" class="cluster">
|
|
||||||
<title>cluster_storage</title>
|
|
||||||
<polygon fill="#f0f0f0" stroke="black" points="8,-15.97 8,-101.97 218,-101.97 218,-15.97 8,-15.97"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="113" y="-82.77" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">File Storage</text>
|
|
||||||
</g>
|
|
||||||
<!-- browser -->
|
|
||||||
<g id="node1" class="node">
|
|
||||||
<title>browser</title>
|
|
||||||
<ellipse fill="none" stroke="black" cx="555" cy="-847.74" rx="69.12" ry="30.05"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-851.69" font-family="Helvetica,sans-Serif" font-size="14.00">Browser</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-834.44" font-family="Helvetica,sans-Serif" font-size="14.00">mpr.local.ar</text>
|
|
||||||
</g>
|
|
||||||
<!-- nginx -->
|
|
||||||
<g id="node2" class="node">
|
|
||||||
<title>nginx</title>
|
|
||||||
<path fill="none" stroke="black" d="M576.5,-744.19C576.5,-744.19 533.5,-744.19 533.5,-744.19 527.5,-744.19 521.5,-738.19 521.5,-732.19 521.5,-732.19 521.5,-713.69 521.5,-713.69 521.5,-707.69 527.5,-701.69 533.5,-701.69 533.5,-701.69 576.5,-701.69 576.5,-701.69 582.5,-701.69 588.5,-707.69 588.5,-713.69 588.5,-713.69 588.5,-732.19 588.5,-732.19 588.5,-738.19 582.5,-744.19 576.5,-744.19"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-726.89" font-family="Helvetica,sans-Serif" font-size="14.00">nginx</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-709.64" font-family="Helvetica,sans-Serif" font-size="14.00">port 80</text>
|
|
||||||
</g>
|
|
||||||
<!-- browser->nginx -->
|
|
||||||
<g id="edge1" class="edge">
|
|
||||||
<title>browser->nginx</title>
|
|
||||||
<path fill="none" stroke="black" d="M555,-817.21C555,-817.21 555,-756.06 555,-756.06"/>
|
|
||||||
<polygon fill="black" stroke="black" points="558.5,-756.06 555,-746.06 551.5,-756.06 558.5,-756.06"/>
|
|
||||||
</g>
|
|
||||||
<!-- django -->
|
|
||||||
<g id="node3" class="node">
|
|
||||||
<title>django</title>
|
|
||||||
<path fill="none" stroke="black" d="M585.5,-616.44C585.5,-616.44 524.5,-616.44 524.5,-616.44 518.5,-616.44 512.5,-610.44 512.5,-604.44 512.5,-604.44 512.5,-568.69 512.5,-568.69 512.5,-562.69 518.5,-556.69 524.5,-556.69 524.5,-556.69 585.5,-556.69 585.5,-556.69 591.5,-556.69 597.5,-562.69 597.5,-568.69 597.5,-568.69 597.5,-604.44 597.5,-604.44 597.5,-610.44 591.5,-616.44 585.5,-616.44"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-599.14" font-family="Helvetica,sans-Serif" font-size="14.00">Django</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-581.89" font-family="Helvetica,sans-Serif" font-size="14.00">/admin</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="555" y="-564.64" font-family="Helvetica,sans-Serif" font-size="14.00">port 8701</text>
|
|
||||||
</g>
|
|
||||||
<!-- nginx->django -->
|
|
||||||
<g id="edge2" class="edge">
|
|
||||||
<title>nginx->django</title>
|
|
||||||
<path fill="none" stroke="black" d="M555,-701.33C555,-701.33 555,-628.2 555,-628.2"/>
|
|
||||||
<polygon fill="black" stroke="black" points="558.5,-628.2 555,-618.2 551.5,-628.2 558.5,-628.2"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="571.88" y="-663.19" font-family="Helvetica,sans-Serif" font-size="10.00">/admin</text>
|
|
||||||
</g>
|
|
||||||
<!-- fastapi -->
|
|
||||||
<g id="node4" class="node">
|
|
||||||
<title>fastapi</title>
|
|
||||||
<path fill="none" stroke="black" d="M554.5,-485.94C554.5,-485.94 493.5,-485.94 493.5,-485.94 487.5,-485.94 481.5,-479.94 481.5,-473.94 481.5,-473.94 481.5,-438.19 481.5,-438.19 481.5,-432.19 487.5,-426.19 493.5,-426.19 493.5,-426.19 554.5,-426.19 554.5,-426.19 560.5,-426.19 566.5,-432.19 566.5,-438.19 566.5,-438.19 566.5,-473.94 566.5,-473.94 566.5,-479.94 560.5,-485.94 554.5,-485.94"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="524" y="-468.64" font-family="Helvetica,sans-Serif" font-size="14.00">FastAPI</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="524" y="-451.39" font-family="Helvetica,sans-Serif" font-size="14.00">/api</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="524" y="-434.14" font-family="Helvetica,sans-Serif" font-size="14.00">port 8702</text>
|
|
||||||
</g>
|
|
||||||
<!-- nginx->fastapi -->
|
|
||||||
<g id="edge3" class="edge">
|
|
||||||
<title>nginx->fastapi</title>
|
|
||||||
<path fill="none" stroke="black" d="M521.02,-716C511.47,-716 503.63,-716 503.63,-716 503.63,-716 503.63,-497.9 503.63,-497.9"/>
|
|
||||||
<polygon fill="black" stroke="black" points="507.13,-497.9 503.63,-487.9 500.13,-497.9 507.13,-497.9"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="723" y="-583.44" font-family="Helvetica,sans-Serif" font-size="10.00">/api</text>
|
|
||||||
</g>
|
|
||||||
<!-- timeline -->
|
|
||||||
<g id="node5" class="node">
|
|
||||||
<title>timeline</title>
|
|
||||||
<path fill="none" stroke="black" d="M442,-616.44C442,-616.44 372,-616.44 372,-616.44 366,-616.44 360,-610.44 360,-604.44 360,-604.44 360,-568.69 360,-568.69 360,-562.69 366,-556.69 372,-556.69 372,-556.69 442,-556.69 442,-556.69 448,-556.69 454,-562.69 454,-568.69 454,-568.69 454,-604.44 454,-604.44 454,-610.44 448,-616.44 442,-616.44"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="407" y="-599.14" font-family="Helvetica,sans-Serif" font-size="14.00">Timeline UI</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="407" y="-581.89" font-family="Helvetica,sans-Serif" font-size="14.00">/ui</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="407" y="-564.64" font-family="Helvetica,sans-Serif" font-size="14.00">port 5173</text>
|
|
||||||
</g>
|
|
||||||
<!-- nginx->timeline -->
|
|
||||||
<g id="edge4" class="edge">
|
|
||||||
<title>nginx->timeline</title>
|
|
||||||
<path fill="none" stroke="black" d="M521.05,-730C477.35,-730 407,-730 407,-730 407,-730 407,-628.15 407,-628.15"/>
|
|
||||||
<polygon fill="black" stroke="black" points="410.5,-628.15 407,-618.15 403.5,-628.15 410.5,-628.15"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="450" y="-663.19" font-family="Helvetica,sans-Serif" font-size="10.00">/ui</text>
|
|
||||||
</g>
|
|
||||||
<!-- django->fastapi -->
|
|
||||||
<g id="edge5" class="edge">
|
|
||||||
<title>django->fastapi</title>
|
|
||||||
<path fill="none" stroke="black" d="M539.5,-556.3C539.5,-556.3 539.5,-497.68 539.5,-497.68"/>
|
|
||||||
<polygon fill="black" stroke="black" points="543,-497.68 539.5,-487.68 536,-497.68 543,-497.68"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="561.88" y="-518.19" font-family="Helvetica,sans-Serif" font-size="10.00">job operations</text>
|
|
||||||
</g>
|
|
||||||
<!-- postgres -->
|
|
||||||
<g id="node9" class="node">
|
|
||||||
<title>postgres</title>
|
|
||||||
<path fill="none" stroke="black" d="M691.75,-69.12C691.75,-72.06 670.35,-74.44 644,-74.44 617.65,-74.44 596.25,-72.06 596.25,-69.12 596.25,-69.12 596.25,-21.31 596.25,-21.31 596.25,-18.38 617.65,-16 644,-16 670.35,-16 691.75,-18.38 691.75,-21.31 691.75,-21.31 691.75,-69.12 691.75,-69.12"/>
|
|
||||||
<path fill="none" stroke="black" d="M691.75,-69.12C691.75,-66.19 670.35,-63.81 644,-63.81 617.65,-63.81 596.25,-66.19 596.25,-69.12"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="644" y="-49.17" font-family="Helvetica,sans-Serif" font-size="14.00">PostgreSQL</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="644" y="-31.92" font-family="Helvetica,sans-Serif" font-size="14.00">port 5433</text>
|
|
||||||
</g>
|
|
||||||
<!-- django->postgres -->
|
|
||||||
<g id="edge6" class="edge">
|
|
||||||
<title>django->postgres</title>
|
|
||||||
<path fill="none" stroke="black" d="M597.82,-587C607.63,-587 615.25,-587 615.25,-587 615.25,-587 615.25,-85.86 615.25,-85.86"/>
|
|
||||||
<polygon fill="black" stroke="black" points="618.75,-85.86 615.25,-75.86 611.75,-85.86 618.75,-85.86"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="808.25" y="-303.81" font-family="Helvetica,sans-Serif" font-size="10.00">CRUD only</text>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_server -->
|
|
||||||
<g id="node6" class="node">
|
|
||||||
<title>grpc_server</title>
|
|
||||||
<path fill="none" stroke="black" d="M353.5,-328.19C353.5,-328.19 274.5,-328.19 274.5,-328.19 268.5,-328.19 262.5,-322.19 262.5,-316.19 262.5,-316.19 262.5,-297.69 262.5,-297.69 262.5,-291.69 268.5,-285.69 274.5,-285.69 274.5,-285.69 353.5,-285.69 353.5,-285.69 359.5,-285.69 365.5,-291.69 365.5,-297.69 365.5,-297.69 365.5,-316.19 365.5,-316.19 365.5,-322.19 359.5,-328.19 353.5,-328.19"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="314" y="-310.89" font-family="Helvetica,sans-Serif" font-size="14.00">gRPC Server</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="314" y="-293.64" font-family="Helvetica,sans-Serif" font-size="14.00">port 50051</text>
|
|
||||||
</g>
|
|
||||||
<!-- fastapi->grpc_server -->
|
|
||||||
<g id="edge10" class="edge">
|
|
||||||
<title>fastapi->grpc_server</title>
|
|
||||||
<path fill="none" stroke="black" d="M509.75,-425.9C509.75,-382.34 509.75,-307 509.75,-307 509.75,-307 377.46,-307 377.46,-307"/>
|
|
||||||
<polygon fill="black" stroke="black" points="377.46,-303.5 367.46,-307 377.46,-310.5 377.46,-303.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="398.25" y="-387.69" font-family="Helvetica,sans-Serif" font-size="10.00">gRPC</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="398.25" y="-374.94" font-family="Helvetica,sans-Serif" font-size="10.00">progress streaming</text>
|
|
||||||
</g>
|
|
||||||
<!-- fastapi->postgres -->
|
|
||||||
<g id="edge8" class="edge">
|
|
||||||
<title>fastapi->postgres</title>
|
|
||||||
<path fill="none" stroke="black" d="M552.25,-425.84C552.25,-330.91 552.25,-45 552.25,-45 552.25,-45 584.46,-45 584.46,-45"/>
|
|
||||||
<polygon fill="black" stroke="black" points="584.46,-48.5 594.46,-45 584.46,-41.5 584.46,-48.5"/>
|
|
||||||
</g>
|
|
||||||
<!-- redis -->
|
|
||||||
<g id="node10" class="node">
|
|
||||||
<title>redis</title>
|
|
||||||
<path fill="none" stroke="black" d="M415.5,-69.12C415.5,-72.06 396.45,-74.44 373,-74.44 349.55,-74.44 330.5,-72.06 330.5,-69.12 330.5,-69.12 330.5,-21.31 330.5,-21.31 330.5,-18.38 349.55,-16 373,-16 396.45,-16 415.5,-18.38 415.5,-21.31 415.5,-21.31 415.5,-69.12 415.5,-69.12"/>
|
|
||||||
<path fill="none" stroke="black" d="M415.5,-69.12C415.5,-66.19 396.45,-63.81 373,-63.81 349.55,-63.81 330.5,-66.19 330.5,-69.12"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="373" y="-49.17" font-family="Helvetica,sans-Serif" font-size="14.00">Redis</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="373" y="-31.92" font-family="Helvetica,sans-Serif" font-size="14.00">port 6380</text>
|
|
||||||
</g>
|
|
||||||
<!-- fastapi->redis -->
|
|
||||||
<g id="edge9" class="edge">
|
|
||||||
<title>fastapi->redis</title>
|
|
||||||
<path fill="none" stroke="black" d="M481.02,-456C442,-456 390.5,-456 390.5,-456 390.5,-456 390.5,-86.27 390.5,-86.27"/>
|
|
||||||
<polygon fill="black" stroke="black" points="394,-86.27 390.5,-76.27 387,-86.27 394,-86.27"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="542" y="-240.81" font-family="Helvetica,sans-Serif" font-size="10.00">job status</text>
|
|
||||||
</g>
|
|
||||||
<!-- timeline->fastapi -->
|
|
||||||
<g id="edge7" class="edge">
|
|
||||||
<title>timeline->fastapi</title>
|
|
||||||
<path fill="none" stroke="black" d="M454.47,-587C475.15,-587 494.75,-587 494.75,-587 494.75,-587 494.75,-497.94 494.75,-497.94"/>
|
|
||||||
<polygon fill="black" stroke="black" points="498.25,-497.94 494.75,-487.94 491.25,-497.94 498.25,-497.94"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="440.75" y="-518.19" font-family="Helvetica,sans-Serif" font-size="10.00">REST API</text>
|
|
||||||
</g>
|
|
||||||
<!-- celery -->
|
|
||||||
<g id="node7" class="node">
|
|
||||||
<title>celery</title>
|
|
||||||
<path fill="none" stroke="black" d="M271.75,-202.19C271.75,-202.19 182.25,-202.19 182.25,-202.19 176.25,-202.19 170.25,-196.19 170.25,-190.19 170.25,-190.19 170.25,-171.69 170.25,-171.69 170.25,-165.69 176.25,-159.69 182.25,-159.69 182.25,-159.69 271.75,-159.69 271.75,-159.69 277.75,-159.69 283.75,-165.69 283.75,-171.69 283.75,-171.69 283.75,-190.19 283.75,-190.19 283.75,-196.19 277.75,-202.19 271.75,-202.19"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="227" y="-184.89" font-family="Helvetica,sans-Serif" font-size="14.00">Celery Worker</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="227" y="-167.64" font-family="Helvetica,sans-Serif" font-size="14.00">(local)</text>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_server->celery -->
|
|
||||||
<g id="edge11" class="edge">
|
|
||||||
<title>grpc_server->celery</title>
|
|
||||||
<path fill="none" stroke="black" d="M269.58,-285.28C269.58,-285.28 269.58,-213.83 269.58,-213.83"/>
|
|
||||||
<polygon fill="black" stroke="black" points="273.08,-213.83 269.58,-203.83 266.08,-213.83 273.08,-213.83"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="223.62" y="-240.81" font-family="Helvetica,sans-Serif" font-size="10.00">task dispatch</text>
|
|
||||||
</g>
|
|
||||||
<!-- celery->grpc_server -->
|
|
||||||
<g id="edge14" class="edge">
|
|
||||||
<title>celery->grpc_server</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M276.67,-202.6C276.67,-202.6 276.67,-274.05 276.67,-274.05"/>
|
|
||||||
<polygon fill="black" stroke="black" points="273.17,-274.05 276.67,-284.05 280.17,-274.05 273.17,-274.05"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="341.88" y="-247.19" font-family="Helvetica,sans-Serif" font-size="10.00">progress</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="341.88" y="-234.44" font-family="Helvetica,sans-Serif" font-size="10.00">callbacks</text>
|
|
||||||
</g>
|
|
||||||
<!-- celery->postgres -->
|
|
||||||
<g id="edge13" class="edge">
|
|
||||||
<title>celery->postgres</title>
|
|
||||||
<path fill="none" stroke="black" d="M284.21,-188C390.19,-188 606.37,-188 606.37,-188 606.37,-188 606.37,-84.94 606.37,-84.94"/>
|
|
||||||
<polygon fill="black" stroke="black" points="609.87,-84.94 606.37,-74.94 602.87,-84.94 609.87,-84.94"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="392.5" y="-121.19" font-family="Helvetica,sans-Serif" font-size="10.00">job updates</text>
|
|
||||||
</g>
|
|
||||||
<!-- celery->redis -->
|
|
||||||
<g id="edge12" class="edge">
|
|
||||||
<title>celery->redis</title>
|
|
||||||
<path fill="none" stroke="black" d="M283.96,-174C315.34,-174 348,-174 348,-174 348,-174 348,-85.95 348,-85.95"/>
|
|
||||||
<polygon fill="black" stroke="black" points="351.5,-85.95 348,-75.95 344.5,-85.95 351.5,-85.95"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="286" y="-121.19" font-family="Helvetica,sans-Serif" font-size="10.00">queue</text>
|
|
||||||
</g>
|
|
||||||
<!-- local_fs -->
|
|
||||||
<g id="node12" class="node">
|
|
||||||
<title>local_fs</title>
|
|
||||||
<polygon fill="none" stroke="black" points="210.12,-66.47 207.12,-70.47 186.12,-70.47 183.12,-66.47 137.88,-66.47 137.88,-23.97 210.12,-23.97 210.12,-66.47"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="174" y="-49.17" font-family="Helvetica,sans-Serif" font-size="14.00">Local FS</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="174" y="-31.92" font-family="Helvetica,sans-Serif" font-size="14.00">/media</text>
|
|
||||||
</g>
|
|
||||||
<!-- celery->local_fs -->
|
|
||||||
<g id="edge15" class="edge">
|
|
||||||
<title>celery->local_fs</title>
|
|
||||||
<path fill="none" stroke="black" d="M190.19,-159.43C190.19,-159.43 190.19,-78.14 190.19,-78.14"/>
|
|
||||||
<polygon fill="black" stroke="black" points="193.69,-78.14 190.19,-68.14 186.69,-78.14 193.69,-78.14"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="182.75" y="-121.19" font-family="Helvetica,sans-Serif" font-size="10.00">read/write</text>
|
|
||||||
</g>
|
|
||||||
<!-- lambda -->
|
|
||||||
<g id="node8" class="node">
|
|
||||||
<title>lambda</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M192.75,-328.19C192.75,-328.19 145.25,-328.19 145.25,-328.19 139.25,-328.19 133.25,-322.19 133.25,-316.19 133.25,-316.19 133.25,-297.69 133.25,-297.69 133.25,-291.69 139.25,-285.69 145.25,-285.69 145.25,-285.69 192.75,-285.69 192.75,-285.69 198.75,-285.69 204.75,-291.69 204.75,-297.69 204.75,-297.69 204.75,-316.19 204.75,-316.19 204.75,-322.19 198.75,-328.19 192.75,-328.19"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="169" y="-310.89" font-family="Helvetica,sans-Serif" font-size="14.00">Lambda</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="169" y="-293.64" font-family="Helvetica,sans-Serif" font-size="14.00">(cloud)</text>
|
|
||||||
</g>
|
|
||||||
<!-- sqs -->
|
|
||||||
<g id="node11" class="node">
|
|
||||||
<title>sqs</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M538,-69.12C538,-72.06 523.66,-74.44 506,-74.44 488.34,-74.44 474,-72.06 474,-69.12 474,-69.12 474,-21.31 474,-21.31 474,-18.38 488.34,-16 506,-16 523.66,-16 538,-18.38 538,-21.31 538,-21.31 538,-69.12 538,-69.12"/>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M538,-69.12C538,-66.19 523.66,-63.81 506,-63.81 488.34,-63.81 474,-66.19 474,-69.12"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="506" y="-49.17" font-family="Helvetica,sans-Serif" font-size="14.00">SQS</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="506" y="-31.92" font-family="Helvetica,sans-Serif" font-size="14.00">(cloud)</text>
|
|
||||||
</g>
|
|
||||||
<!-- lambda->sqs -->
|
|
||||||
<g id="edge16" class="edge">
|
|
||||||
<title>lambda->sqs</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M187.5,-285.28C187.5,-267.07 187.5,-244 187.5,-244 187.5,-244 477.75,-244 477.75,-244 477.75,-244 477.75,-84.37 477.75,-84.37"/>
|
|
||||||
<polygon fill="black" stroke="black" points="481.25,-84.37 477.75,-74.37 474.25,-84.37 481.25,-84.37"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="415" y="-177.81" font-family="Helvetica,sans-Serif" font-size="10.00">queue</text>
|
|
||||||
</g>
|
|
||||||
<!-- s3 -->
|
|
||||||
<g id="node13" class="node">
|
|
||||||
<title>s3</title>
|
|
||||||
<polygon fill="none" stroke="black" stroke-dasharray="5,2" points="80,-66.47 77,-70.47 56,-70.47 53,-66.47 16,-66.47 16,-23.97 80,-23.97 80,-66.47"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="48" y="-49.17" font-family="Helvetica,sans-Serif" font-size="14.00">S3</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="48" y="-31.92" font-family="Helvetica,sans-Serif" font-size="14.00">(cloud)</text>
|
|
||||||
</g>
|
|
||||||
<!-- lambda->s3 -->
|
|
||||||
<g id="edge17" class="edge">
|
|
||||||
<title>lambda->s3</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M133.02,-307C97.36,-307 48,-307 48,-307 48,-307 48,-78.15 48,-78.15"/>
|
|
||||||
<polygon fill="black" stroke="black" points="51.5,-78.15 48,-68.15 44.5,-78.15 51.5,-78.15"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="80.75" y="-177.81" font-family="Helvetica,sans-Serif" font-size="10.00">read/write</text>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 20 KiB |
94
docs/architecture/01a-local-architecture.dot
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
digraph local_architecture {
|
||||||
|
rankdir=TB
|
||||||
|
node [shape=box, style=rounded, fontname="Helvetica"]
|
||||||
|
edge [fontname="Helvetica", fontsize=10]
|
||||||
|
|
||||||
|
labelloc="t"
|
||||||
|
label="MPR - Local Architecture (Celery + MinIO)"
|
||||||
|
fontsize=16
|
||||||
|
fontname="Helvetica-Bold"
|
||||||
|
|
||||||
|
graph [splines=ortho, nodesep=0.8, ranksep=0.8]
|
||||||
|
|
||||||
|
// External
|
||||||
|
subgraph cluster_external {
|
||||||
|
label="External"
|
||||||
|
style=dashed
|
||||||
|
color=gray
|
||||||
|
|
||||||
|
browser [label="Browser\nmpr.local.ar", shape=ellipse]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nginx reverse proxy
|
||||||
|
subgraph cluster_proxy {
|
||||||
|
label="Reverse Proxy"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#e8f4f8"
|
||||||
|
|
||||||
|
nginx [label="nginx\nport 80"]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Application layer
|
||||||
|
subgraph cluster_apps {
|
||||||
|
label="Application Layer"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#f0f8e8"
|
||||||
|
|
||||||
|
django [label="Django Admin\n/admin\nport 8701"]
|
||||||
|
fastapi [label="GraphQL API\n/graphql\nport 8702"]
|
||||||
|
timeline [label="Timeline UI\n/\nport 5173"]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Worker layer
|
||||||
|
subgraph cluster_workers {
|
||||||
|
label="Worker Layer"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#fff8e8"
|
||||||
|
|
||||||
|
grpc_server [label="gRPC Server\nport 50051"]
|
||||||
|
celery [label="Celery Worker\nFFmpeg transcoding"]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Data layer
|
||||||
|
subgraph cluster_data {
|
||||||
|
label="Data Layer"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#f8e8f0"
|
||||||
|
|
||||||
|
postgres [label="PostgreSQL\nport 5436", shape=cylinder]
|
||||||
|
redis [label="Redis\nCelery queue\nport 6381", shape=cylinder]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Storage
|
||||||
|
subgraph cluster_storage {
|
||||||
|
label="S3 Storage (MinIO)"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#f0f0f0"
|
||||||
|
|
||||||
|
minio [label="MinIO\nS3-compatible API\nport 9000", shape=folder]
|
||||||
|
bucket_in [label="mpr-media-in/\ninput videos", shape=note]
|
||||||
|
bucket_out [label="mpr-media-out/\ntranscoded output", shape=note]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connections
|
||||||
|
browser -> nginx [label="HTTP"]
|
||||||
|
|
||||||
|
nginx -> django [xlabel="/admin"]
|
||||||
|
nginx -> fastapi [xlabel="/graphql"]
|
||||||
|
nginx -> timeline [xlabel="/"]
|
||||||
|
nginx -> minio [xlabel="/media/*"]
|
||||||
|
|
||||||
|
timeline -> fastapi [label="GraphQL"]
|
||||||
|
django -> postgres
|
||||||
|
|
||||||
|
fastapi -> postgres [label="read/write jobs"]
|
||||||
|
fastapi -> grpc_server [label="gRPC\nprogress updates"]
|
||||||
|
|
||||||
|
grpc_server -> celery [label="dispatch tasks"]
|
||||||
|
celery -> redis [label="task queue"]
|
||||||
|
celery -> postgres [label="update job status"]
|
||||||
|
celery -> minio [label="S3 API\ndownload input\nupload output"]
|
||||||
|
|
||||||
|
minio -> bucket_in [style=dotted, arrowhead=none]
|
||||||
|
minio -> bucket_out [style=dotted, arrowhead=none]
|
||||||
|
}
|
||||||
242
docs/architecture/01a-local-architecture.svg
Normal file
@@ -0,0 +1,242 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
|
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||||
|
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||||
|
<!-- Generated by graphviz version 14.1.2 (0)
|
||||||
|
-->
|
||||||
|
<!-- Title: local_architecture Pages: 1 -->
|
||||||
|
<svg width="667pt" height="1095pt"
|
||||||
|
viewBox="0.00 0.00 667.00 1095.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||||
|
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1090.76)">
|
||||||
|
<title>local_architecture</title>
|
||||||
|
<polygon fill="white" stroke="none" points="-4,4 -4,-1090.76 663,-1090.76 663,4 -4,4"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="329.5" y="-1067.56" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - Local Architecture (Celery + MinIO)</text>
|
||||||
|
<g id="clust1" class="cluster">
|
||||||
|
<title>cluster_external</title>
|
||||||
|
<polygon fill="none" stroke="gray" stroke-dasharray="5,2" points="270,-947.66 270,-1051.26 424,-1051.26 424,-947.66 270,-947.66"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="347" y="-1032.06" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">External</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust2" class="cluster">
|
||||||
|
<title>cluster_proxy</title>
|
||||||
|
<polygon fill="#e8f4f8" stroke="black" points="274,-819.91 274,-905.91 420,-905.91 420,-819.91 274,-819.91"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="347" y="-886.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Reverse Proxy</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust3" class="cluster">
|
||||||
|
<title>cluster_apps</title>
|
||||||
|
<polygon fill="#f0f8e8" stroke="black" points="19,-556.16 19,-789.91 301,-789.91 301,-556.16 19,-556.16"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="160" y="-770.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Application Layer</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust4" class="cluster">
|
||||||
|
<title>cluster_workers</title>
|
||||||
|
<polygon fill="#fff8e8" stroke="black" points="193,-302.41 193,-501.66 369,-501.66 369,-302.41 193,-302.41"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="281" y="-482.46" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Worker Layer</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust5" class="cluster">
|
||||||
|
<title>cluster_data</title>
|
||||||
|
<polygon fill="#f8e8f0" stroke="black" points="8,-109.5 8,-235.16 286,-235.16 286,-109.5 8,-109.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="147" y="-215.96" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Data Layer</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust6" class="cluster">
|
||||||
|
<title>cluster_storage</title>
|
||||||
|
<polygon fill="#f0f0f0" stroke="black" points="319,-8 319,-223.95 651,-223.95 651,-8 319,-8"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="485" y="-204.75" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">S3 Storage (MinIO)</text>
|
||||||
|
</g>
|
||||||
|
<!-- browser -->
|
||||||
|
<g id="node1" class="node">
|
||||||
|
<title>browser</title>
|
||||||
|
<ellipse fill="none" stroke="black" cx="347" cy="-985.71" rx="69.12" ry="30.05"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="347" y="-989.66" font-family="Helvetica,sans-Serif" font-size="14.00">Browser</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="347" y="-972.41" font-family="Helvetica,sans-Serif" font-size="14.00">mpr.local.ar</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx -->
|
||||||
|
<g id="node2" class="node">
|
||||||
|
<title>nginx</title>
|
||||||
|
<path fill="none" stroke="black" d="M368.5,-870.41C368.5,-870.41 325.5,-870.41 325.5,-870.41 319.5,-870.41 313.5,-864.41 313.5,-858.41 313.5,-858.41 313.5,-839.91 313.5,-839.91 313.5,-833.91 319.5,-827.91 325.5,-827.91 325.5,-827.91 368.5,-827.91 368.5,-827.91 374.5,-827.91 380.5,-833.91 380.5,-839.91 380.5,-839.91 380.5,-858.41 380.5,-858.41 380.5,-864.41 374.5,-870.41 368.5,-870.41"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="347" y="-853.11" font-family="Helvetica,sans-Serif" font-size="14.00">nginx</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="347" y="-835.86" font-family="Helvetica,sans-Serif" font-size="14.00">port 80</text>
|
||||||
|
</g>
|
||||||
|
<!-- browser->nginx -->
|
||||||
|
<g id="edge1" class="edge">
|
||||||
|
<title>browser->nginx</title>
|
||||||
|
<path fill="none" stroke="black" d="M347,-955.4C347,-955.4 347,-882.41 347,-882.41"/>
|
||||||
|
<polygon fill="black" stroke="black" points="350.5,-882.41 347,-872.41 343.5,-882.41 350.5,-882.41"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="359.75" y="-917.16" font-family="Helvetica,sans-Serif" font-size="10.00">HTTP</text>
|
||||||
|
</g>
|
||||||
|
<!-- django -->
|
||||||
|
<g id="node3" class="node">
|
||||||
|
<title>django</title>
|
||||||
|
<path fill="none" stroke="black" d="M128.75,-754.41C128.75,-754.41 39.25,-754.41 39.25,-754.41 33.25,-754.41 27.25,-748.41 27.25,-742.41 27.25,-742.41 27.25,-706.66 27.25,-706.66 27.25,-700.66 33.25,-694.66 39.25,-694.66 39.25,-694.66 128.75,-694.66 128.75,-694.66 134.75,-694.66 140.75,-700.66 140.75,-706.66 140.75,-706.66 140.75,-742.41 140.75,-742.41 140.75,-748.41 134.75,-754.41 128.75,-754.41"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-737.11" font-family="Helvetica,sans-Serif" font-size="14.00">Django Admin</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-719.86" font-family="Helvetica,sans-Serif" font-size="14.00">/admin</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-702.61" font-family="Helvetica,sans-Serif" font-size="14.00">port 8701</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->django -->
|
||||||
|
<g id="edge2" class="edge">
|
||||||
|
<title>nginx->django</title>
|
||||||
|
<path fill="none" stroke="black" d="M313.16,-856C242.12,-856 84,-856 84,-856 84,-856 84,-766.21 84,-766.21"/>
|
||||||
|
<polygon fill="black" stroke="black" points="87.5,-766.21 84,-756.21 80.5,-766.21 87.5,-766.21"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="136.81" y="-859.25" font-family="Helvetica,sans-Serif" font-size="10.00">/admin</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi -->
|
||||||
|
<g id="node4" class="node">
|
||||||
|
<title>fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M281.25,-623.91C281.25,-623.91 200.75,-623.91 200.75,-623.91 194.75,-623.91 188.75,-617.91 188.75,-611.91 188.75,-611.91 188.75,-576.16 188.75,-576.16 188.75,-570.16 194.75,-564.16 200.75,-564.16 200.75,-564.16 281.25,-564.16 281.25,-564.16 287.25,-564.16 293.25,-570.16 293.25,-576.16 293.25,-576.16 293.25,-611.91 293.25,-611.91 293.25,-617.91 287.25,-623.91 281.25,-623.91"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="241" y="-606.61" font-family="Helvetica,sans-Serif" font-size="14.00">GraphQL API</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="241" y="-589.36" font-family="Helvetica,sans-Serif" font-size="14.00">/graphql</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="241" y="-572.11" font-family="Helvetica,sans-Serif" font-size="14.00">port 8702</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->fastapi -->
|
||||||
|
<g id="edge3" class="edge">
|
||||||
|
<title>nginx->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M337.06,-827.84C337.06,-766.52 337.06,-594 337.06,-594 337.06,-594 305.04,-594 305.04,-594"/>
|
||||||
|
<polygon fill="black" stroke="black" points="305.04,-590.5 295.04,-594 305.04,-597.5 305.04,-590.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="317.19" y="-698.16" font-family="Helvetica,sans-Serif" font-size="10.00">/graphql</text>
|
||||||
|
</g>
|
||||||
|
<!-- timeline -->
|
||||||
|
<g id="node5" class="node">
|
||||||
|
<title>timeline</title>
|
||||||
|
<path fill="none" stroke="black" d="M281,-754.41C281,-754.41 211,-754.41 211,-754.41 205,-754.41 199,-748.41 199,-742.41 199,-742.41 199,-706.66 199,-706.66 199,-700.66 205,-694.66 211,-694.66 211,-694.66 281,-694.66 281,-694.66 287,-694.66 293,-700.66 293,-706.66 293,-706.66 293,-742.41 293,-742.41 293,-748.41 287,-754.41 281,-754.41"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="246" y="-737.11" font-family="Helvetica,sans-Serif" font-size="14.00">Timeline UI</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="246" y="-719.86" font-family="Helvetica,sans-Serif" font-size="14.00">/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="246" y="-702.61" font-family="Helvetica,sans-Serif" font-size="14.00">port 5173</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->timeline -->
|
||||||
|
<g id="edge4" class="edge">
|
||||||
|
<title>nginx->timeline</title>
|
||||||
|
<path fill="none" stroke="black" d="M313.34,-842C298.97,-842 285.44,-842 285.44,-842 285.44,-842 285.44,-766.3 285.44,-766.3"/>
|
||||||
|
<polygon fill="black" stroke="black" points="288.94,-766.3 285.44,-756.3 281.94,-766.3 288.94,-766.3"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="283.94" y="-821.35" font-family="Helvetica,sans-Serif" font-size="10.00">/</text>
|
||||||
|
</g>
|
||||||
|
<!-- minio -->
|
||||||
|
<g id="node10" class="node">
|
||||||
|
<title>minio</title>
|
||||||
|
<polygon fill="none" stroke="black" points="486.38,-188.45 483.38,-192.45 462.38,-192.45 459.38,-188.45 343.62,-188.45 343.62,-128.7 486.38,-128.7 486.38,-188.45"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="415" y="-171.15" font-family="Helvetica,sans-Serif" font-size="14.00">MinIO</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="415" y="-153.9" font-family="Helvetica,sans-Serif" font-size="14.00">S3-compatible API</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="415" y="-136.65" font-family="Helvetica,sans-Serif" font-size="14.00">port 9000</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->minio -->
|
||||||
|
<g id="edge5" class="edge">
|
||||||
|
<title>nginx->minio</title>
|
||||||
|
<path fill="none" stroke="black" d="M370.56,-827.73C370.56,-827.73 370.56,-200.13 370.56,-200.13"/>
|
||||||
|
<polygon fill="black" stroke="black" points="374.06,-200.13 370.56,-190.13 367.06,-200.13 374.06,-200.13"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="391.56" y="-517.18" font-family="Helvetica,sans-Serif" font-size="10.00">/media/*</text>
|
||||||
|
</g>
|
||||||
|
<!-- postgres -->
|
||||||
|
<g id="node8" class="node">
|
||||||
|
<title>postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M111.75,-182.48C111.75,-185.42 90.35,-187.8 64,-187.8 37.65,-187.8 16.25,-185.42 16.25,-182.48 16.25,-182.48 16.25,-134.67 16.25,-134.67 16.25,-131.74 37.65,-129.36 64,-129.36 90.35,-129.36 111.75,-131.74 111.75,-134.67 111.75,-134.67 111.75,-182.48 111.75,-182.48"/>
|
||||||
|
<path fill="none" stroke="black" d="M111.75,-182.48C111.75,-179.55 90.35,-177.17 64,-177.17 37.65,-177.17 16.25,-179.55 16.25,-182.48"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="64" y="-162.53" font-family="Helvetica,sans-Serif" font-size="14.00">PostgreSQL</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="64" y="-145.28" font-family="Helvetica,sans-Serif" font-size="14.00">port 5436</text>
|
||||||
|
</g>
|
||||||
|
<!-- django->postgres -->
|
||||||
|
<g id="edge7" class="edge">
|
||||||
|
<title>django->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M48.38,-694.5C48.38,-694.5 48.38,-199.71 48.38,-199.71"/>
|
||||||
|
<polygon fill="black" stroke="black" points="51.88,-199.71 48.38,-189.71 44.88,-199.71 51.88,-199.71"/>
|
||||||
|
</g>
|
||||||
|
<!-- grpc_server -->
|
||||||
|
<g id="node6" class="node">
|
||||||
|
<title>grpc_server</title>
|
||||||
|
<path fill="none" stroke="black" d="M301.5,-466.16C301.5,-466.16 222.5,-466.16 222.5,-466.16 216.5,-466.16 210.5,-460.16 210.5,-454.16 210.5,-454.16 210.5,-435.66 210.5,-435.66 210.5,-429.66 216.5,-423.66 222.5,-423.66 222.5,-423.66 301.5,-423.66 301.5,-423.66 307.5,-423.66 313.5,-429.66 313.5,-435.66 313.5,-435.66 313.5,-454.16 313.5,-454.16 313.5,-460.16 307.5,-466.16 301.5,-466.16"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="262" y="-448.86" font-family="Helvetica,sans-Serif" font-size="14.00">gRPC Server</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="262" y="-431.61" font-family="Helvetica,sans-Serif" font-size="14.00">port 50051</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->grpc_server -->
|
||||||
|
<g id="edge9" class="edge">
|
||||||
|
<title>fastapi->grpc_server</title>
|
||||||
|
<path fill="none" stroke="black" d="M251.88,-563.85C251.88,-563.85 251.88,-477.88 251.88,-477.88"/>
|
||||||
|
<polygon fill="black" stroke="black" points="255.38,-477.88 251.88,-467.88 248.38,-477.88 255.38,-477.88"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="292" y="-525.66" font-family="Helvetica,sans-Serif" font-size="10.00">gRPC</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="292" y="-512.91" font-family="Helvetica,sans-Serif" font-size="10.00">progress updates</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->postgres -->
|
||||||
|
<g id="edge8" class="edge">
|
||||||
|
<title>fastapi->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M188.61,-594C138.18,-594 69.5,-594 69.5,-594 69.5,-594 69.5,-199.68 69.5,-199.68"/>
|
||||||
|
<polygon fill="black" stroke="black" points="73,-199.68 69.5,-189.68 66,-199.68 73,-199.68"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="82.38" y="-385.16" font-family="Helvetica,sans-Serif" font-size="10.00">read/write jobs</text>
|
||||||
|
</g>
|
||||||
|
<!-- timeline->fastapi -->
|
||||||
|
<g id="edge6" class="edge">
|
||||||
|
<title>timeline->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M246,-694.26C246,-694.26 246,-635.65 246,-635.65"/>
|
||||||
|
<polygon fill="black" stroke="black" points="249.5,-635.65 246,-625.65 242.5,-635.65 249.5,-635.65"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="264" y="-656.16" font-family="Helvetica,sans-Serif" font-size="10.00">GraphQL</text>
|
||||||
|
</g>
|
||||||
|
<!-- celery -->
|
||||||
|
<g id="node7" class="node">
|
||||||
|
<title>celery</title>
|
||||||
|
<path fill="none" stroke="black" d="M348.62,-352.91C348.62,-352.91 213.38,-352.91 213.38,-352.91 207.38,-352.91 201.38,-346.91 201.38,-340.91 201.38,-340.91 201.38,-322.41 201.38,-322.41 201.38,-316.41 207.38,-310.41 213.38,-310.41 213.38,-310.41 348.62,-310.41 348.62,-310.41 354.62,-310.41 360.62,-316.41 360.62,-322.41 360.62,-322.41 360.62,-340.91 360.62,-340.91 360.62,-346.91 354.62,-352.91 348.62,-352.91"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="281" y="-335.61" font-family="Helvetica,sans-Serif" font-size="14.00">Celery Worker</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="281" y="-318.36" font-family="Helvetica,sans-Serif" font-size="14.00">FFmpeg transcoding</text>
|
||||||
|
</g>
|
||||||
|
<!-- grpc_server->celery -->
|
||||||
|
<g id="edge10" class="edge">
|
||||||
|
<title>grpc_server->celery</title>
|
||||||
|
<path fill="none" stroke="black" d="M262,-423.34C262,-423.34 262,-364.66 262,-364.66"/>
|
||||||
|
<polygon fill="black" stroke="black" points="265.5,-364.66 262,-354.66 258.5,-364.66 265.5,-364.66"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="305.25" y="-385.16" font-family="Helvetica,sans-Serif" font-size="10.00">dispatch tasks</text>
|
||||||
|
</g>
|
||||||
|
<!-- celery->postgres -->
|
||||||
|
<g id="edge12" class="edge">
|
||||||
|
<title>celery->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M201.09,-332C148.99,-332 90.62,-332 90.62,-332 90.62,-332 90.62,-199.51 90.62,-199.51"/>
|
||||||
|
<polygon fill="black" stroke="black" points="94.13,-199.51 90.63,-189.51 87.13,-199.51 94.13,-199.51"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="181.38" y="-259.16" font-family="Helvetica,sans-Serif" font-size="10.00">update job status</text>
|
||||||
|
</g>
|
||||||
|
<!-- redis -->
|
||||||
|
<g id="node9" class="node">
|
||||||
|
<title>redis</title>
|
||||||
|
<path fill="none" stroke="black" d="M278.12,-192.19C278.12,-196.31 253.87,-199.66 224,-199.66 194.13,-199.66 169.88,-196.31 169.88,-192.19 169.88,-192.19 169.88,-124.97 169.88,-124.97 169.88,-120.85 194.13,-117.5 224,-117.5 253.87,-117.5 278.12,-120.85 278.12,-124.97 278.12,-124.97 278.12,-192.19 278.12,-192.19"/>
|
||||||
|
<path fill="none" stroke="black" d="M278.12,-192.19C278.12,-188.07 253.87,-184.72 224,-184.72 194.13,-184.72 169.88,-188.07 169.88,-192.19"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="224" y="-171.15" font-family="Helvetica,sans-Serif" font-size="14.00">Redis</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="224" y="-153.9" font-family="Helvetica,sans-Serif" font-size="14.00">Celery queue</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="224" y="-136.65" font-family="Helvetica,sans-Serif" font-size="14.00">port 6381</text>
|
||||||
|
</g>
|
||||||
|
<!-- celery->redis -->
|
||||||
|
<g id="edge11" class="edge">
|
||||||
|
<title>celery->redis</title>
|
||||||
|
<path fill="none" stroke="black" d="M239.75,-310.09C239.75,-310.09 239.75,-211.49 239.75,-211.49"/>
|
||||||
|
<polygon fill="black" stroke="black" points="243.25,-211.49 239.75,-201.49 236.25,-211.49 243.25,-211.49"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="314" y="-259.16" font-family="Helvetica,sans-Serif" font-size="10.00">task queue</text>
|
||||||
|
</g>
|
||||||
|
<!-- celery->minio -->
|
||||||
|
<g id="edge13" class="edge">
|
||||||
|
<title>celery->minio</title>
|
||||||
|
<path fill="none" stroke="black" d="M352.12,-310.09C352.12,-310.09 352.12,-200.39 352.12,-200.39"/>
|
||||||
|
<polygon fill="black" stroke="black" points="355.63,-200.39 352.13,-190.39 348.63,-200.39 355.63,-200.39"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="441.5" y="-271.91" font-family="Helvetica,sans-Serif" font-size="10.00">S3 API</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="441.5" y="-259.16" font-family="Helvetica,sans-Serif" font-size="10.00">download input</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="441.5" y="-246.41" font-family="Helvetica,sans-Serif" font-size="10.00">upload output</text>
|
||||||
|
</g>
|
||||||
|
<!-- bucket_in -->
|
||||||
|
<g id="node11" class="node">
|
||||||
|
<title>bucket_in</title>
|
||||||
|
<polygon fill="none" stroke="black" points="434.75,-58.5 327.25,-58.5 327.25,-16 440.75,-16 440.75,-52.5 434.75,-58.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="434.75,-58.5 434.75,-52.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="440.75,-52.5 434.75,-52.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="384" y="-41.2" font-family="Helvetica,sans-Serif" font-size="14.00">mpr-media-in/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="384" y="-23.95" font-family="Helvetica,sans-Serif" font-size="14.00">input videos</text>
|
||||||
|
</g>
|
||||||
|
<!-- minio->bucket_in -->
|
||||||
|
<g id="edge14" class="edge">
|
||||||
|
<title>minio->bucket_in</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M392.19,-128.27C392.19,-106.66 392.19,-78.11 392.19,-58.79"/>
|
||||||
|
</g>
|
||||||
|
<!-- bucket_out -->
|
||||||
|
<g id="node12" class="node">
|
||||||
|
<title>bucket_out</title>
|
||||||
|
<polygon fill="none" stroke="black" points="637.12,-58.5 498.88,-58.5 498.88,-16 643.12,-16 643.12,-52.5 637.12,-58.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="637.12,-58.5 637.12,-52.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="643.12,-52.5 637.12,-52.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="571" y="-41.2" font-family="Helvetica,sans-Serif" font-size="14.00">mpr-media-out/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="571" y="-23.95" font-family="Helvetica,sans-Serif" font-size="14.00">transcoded output</text>
|
||||||
|
</g>
|
||||||
|
<!-- minio->bucket_out -->
|
||||||
|
<g id="edge15" class="edge">
|
||||||
|
<title>minio->bucket_out</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M463.56,-128.21C463.56,-92.2 463.56,-37 463.56,-37 463.56,-37 479.15,-37 498.44,-37"/>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 18 KiB |
85
docs/architecture/01b-aws-architecture.dot
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
digraph aws_architecture {
|
||||||
|
rankdir=TB
|
||||||
|
node [shape=box, style=rounded, fontname="Helvetica"]
|
||||||
|
edge [fontname="Helvetica", fontsize=10]
|
||||||
|
|
||||||
|
labelloc="t"
|
||||||
|
label="MPR - AWS Architecture (Lambda + Step Functions)"
|
||||||
|
fontsize=16
|
||||||
|
fontname="Helvetica-Bold"
|
||||||
|
|
||||||
|
graph [splines=ortho, nodesep=0.8, ranksep=0.8]
|
||||||
|
|
||||||
|
// External
|
||||||
|
subgraph cluster_external {
|
||||||
|
label="External"
|
||||||
|
style=dashed
|
||||||
|
color=gray
|
||||||
|
|
||||||
|
browser [label="Browser\nmpr.mcrn.ar", shape=ellipse]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nginx reverse proxy
|
||||||
|
subgraph cluster_proxy {
|
||||||
|
label="Reverse Proxy"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#e8f4f8"
|
||||||
|
|
||||||
|
nginx [label="nginx\nport 80"]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Application layer
|
||||||
|
subgraph cluster_apps {
|
||||||
|
label="Application Layer"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#f0f8e8"
|
||||||
|
|
||||||
|
django [label="Django Admin\n/admin\nport 8701"]
|
||||||
|
fastapi [label="GraphQL API\n/graphql\nport 8702"]
|
||||||
|
timeline [label="Timeline UI\n/\nport 5173"]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Data layer (still local)
|
||||||
|
subgraph cluster_data {
|
||||||
|
label="Data Layer"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#f8e8f0"
|
||||||
|
|
||||||
|
postgres [label="PostgreSQL\nport 5436", shape=cylinder]
|
||||||
|
}
|
||||||
|
|
||||||
|
// AWS layer
|
||||||
|
subgraph cluster_aws {
|
||||||
|
label="AWS Cloud"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#fde8d0"
|
||||||
|
|
||||||
|
step_functions [label="Step Functions\nOrchestration\nstate machine"]
|
||||||
|
lambda [label="Lambda Function\nFFmpeg container\ntranscoding"]
|
||||||
|
s3 [label="S3 Buckets", shape=folder]
|
||||||
|
bucket_in [label="mpr-media-in/\ninput videos", shape=note]
|
||||||
|
bucket_out [label="mpr-media-out/\ntranscoded output", shape=note]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connections
|
||||||
|
browser -> nginx [label="HTTP"]
|
||||||
|
|
||||||
|
nginx -> django [xlabel="/admin"]
|
||||||
|
nginx -> fastapi [xlabel="/graphql"]
|
||||||
|
nginx -> timeline [xlabel="/"]
|
||||||
|
|
||||||
|
timeline -> fastapi [label="GraphQL"]
|
||||||
|
django -> postgres
|
||||||
|
|
||||||
|
fastapi -> postgres [label="read/write jobs"]
|
||||||
|
fastapi -> step_functions [label="boto3\nstart_execution()\nexecution_arn"]
|
||||||
|
|
||||||
|
step_functions -> lambda [label="invoke with\njob parameters"]
|
||||||
|
lambda -> s3 [label="download input\nupload output"]
|
||||||
|
lambda -> fastapi [label="POST /jobs/{id}/callback\nupdate status"]
|
||||||
|
|
||||||
|
fastapi -> postgres [label="callback updates\njob status"]
|
||||||
|
|
||||||
|
s3 -> bucket_in [style=dotted, arrowhead=none]
|
||||||
|
s3 -> bucket_out [style=dotted, arrowhead=none]
|
||||||
|
}
|
||||||
224
docs/architecture/01b-aws-architecture.svg
Normal file
@@ -0,0 +1,224 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
|
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||||
|
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||||
|
<!-- Generated by graphviz version 14.1.2 (0)
|
||||||
|
-->
|
||||||
|
<!-- Title: aws_architecture Pages: 1 -->
|
||||||
|
<svg width="639pt" height="1081pt"
|
||||||
|
viewBox="0.00 0.00 639.00 1081.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||||
|
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1077.35)">
|
||||||
|
<title>aws_architecture</title>
|
||||||
|
<polygon fill="white" stroke="none" points="-4,4 -4,-1077.35 635.25,-1077.35 635.25,4 -4,4"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="315.62" y="-1054.15" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - AWS Architecture (Lambda + Step Functions)</text>
|
||||||
|
<g id="clust1" class="cluster">
|
||||||
|
<title>cluster_external</title>
|
||||||
|
<polygon fill="none" stroke="gray" stroke-dasharray="5,2" points="155,-934.25 155,-1037.85 315,-1037.85 315,-934.25 155,-934.25"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-1018.65" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">External</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust2" class="cluster">
|
||||||
|
<title>cluster_proxy</title>
|
||||||
|
<polygon fill="#e8f4f8" stroke="black" points="162,-806.5 162,-892.5 308,-892.5 308,-806.5 162,-806.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-873.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Reverse Proxy</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust3" class="cluster">
|
||||||
|
<title>cluster_apps</title>
|
||||||
|
<polygon fill="#f0f8e8" stroke="black" points="8,-542.75 8,-776.5 290,-776.5 290,-542.75 8,-542.75"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="149" y="-757.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Application Layer</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust4" class="cluster">
|
||||||
|
<title>cluster_data</title>
|
||||||
|
<polygon fill="#f8e8f0" stroke="black" points="27,-372.91 27,-474.84 141,-474.84 141,-372.91 27,-372.91"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-455.64" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Data Layer</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust5" class="cluster">
|
||||||
|
<title>cluster_aws</title>
|
||||||
|
<polygon fill="#fde8d0" stroke="black" points="264,-8 264,-475.5 596,-475.5 596,-8 264,-8"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="430" y="-456.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">AWS Cloud</text>
|
||||||
|
</g>
|
||||||
|
<!-- browser -->
|
||||||
|
<g id="node1" class="node">
|
||||||
|
<title>browser</title>
|
||||||
|
<ellipse fill="none" stroke="black" cx="235" cy="-972.3" rx="71.77" ry="30.05"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-976.25" font-family="Helvetica,sans-Serif" font-size="14.00">Browser</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-959" font-family="Helvetica,sans-Serif" font-size="14.00">mpr.mcrn.ar</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx -->
|
||||||
|
<g id="node2" class="node">
|
||||||
|
<title>nginx</title>
|
||||||
|
<path fill="none" stroke="black" d="M256.5,-857C256.5,-857 213.5,-857 213.5,-857 207.5,-857 201.5,-851 201.5,-845 201.5,-845 201.5,-826.5 201.5,-826.5 201.5,-820.5 207.5,-814.5 213.5,-814.5 213.5,-814.5 256.5,-814.5 256.5,-814.5 262.5,-814.5 268.5,-820.5 268.5,-826.5 268.5,-826.5 268.5,-845 268.5,-845 268.5,-851 262.5,-857 256.5,-857"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-839.7" font-family="Helvetica,sans-Serif" font-size="14.00">nginx</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-822.45" font-family="Helvetica,sans-Serif" font-size="14.00">port 80</text>
|
||||||
|
</g>
|
||||||
|
<!-- browser->nginx -->
|
||||||
|
<g id="edge1" class="edge">
|
||||||
|
<title>browser->nginx</title>
|
||||||
|
<path fill="none" stroke="black" d="M235,-942C235,-942 235,-869 235,-869"/>
|
||||||
|
<polygon fill="black" stroke="black" points="238.5,-869 235,-859 231.5,-869 238.5,-869"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="247.75" y="-903.75" font-family="Helvetica,sans-Serif" font-size="10.00">HTTP</text>
|
||||||
|
</g>
|
||||||
|
<!-- django -->
|
||||||
|
<g id="node3" class="node">
|
||||||
|
<title>django</title>
|
||||||
|
<path fill="none" stroke="black" d="M117.75,-741C117.75,-741 28.25,-741 28.25,-741 22.25,-741 16.25,-735 16.25,-729 16.25,-729 16.25,-693.25 16.25,-693.25 16.25,-687.25 22.25,-681.25 28.25,-681.25 28.25,-681.25 117.75,-681.25 117.75,-681.25 123.75,-681.25 129.75,-687.25 129.75,-693.25 129.75,-693.25 129.75,-729 129.75,-729 129.75,-735 123.75,-741 117.75,-741"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="73" y="-723.7" font-family="Helvetica,sans-Serif" font-size="14.00">Django Admin</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="73" y="-706.45" font-family="Helvetica,sans-Serif" font-size="14.00">/admin</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="73" y="-689.2" font-family="Helvetica,sans-Serif" font-size="14.00">port 8701</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->django -->
|
||||||
|
<g id="edge2" class="edge">
|
||||||
|
<title>nginx->django</title>
|
||||||
|
<path fill="none" stroke="black" d="M201.04,-843C153.54,-843 73,-843 73,-843 73,-843 73,-752.89 73,-752.89"/>
|
||||||
|
<polygon fill="black" stroke="black" points="76.5,-752.89 73,-742.89 69.5,-752.89 76.5,-752.89"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="75.09" y="-846.25" font-family="Helvetica,sans-Serif" font-size="10.00">/admin</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi -->
|
||||||
|
<g id="node4" class="node">
|
||||||
|
<title>fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M270.25,-610.5C270.25,-610.5 189.75,-610.5 189.75,-610.5 183.75,-610.5 177.75,-604.5 177.75,-598.5 177.75,-598.5 177.75,-562.75 177.75,-562.75 177.75,-556.75 183.75,-550.75 189.75,-550.75 189.75,-550.75 270.25,-550.75 270.25,-550.75 276.25,-550.75 282.25,-556.75 282.25,-562.75 282.25,-562.75 282.25,-598.5 282.25,-598.5 282.25,-604.5 276.25,-610.5 270.25,-610.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="230" y="-593.2" font-family="Helvetica,sans-Serif" font-size="14.00">GraphQL API</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="230" y="-575.95" font-family="Helvetica,sans-Serif" font-size="14.00">/graphql</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="230" y="-558.7" font-family="Helvetica,sans-Serif" font-size="14.00">port 8702</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->fastapi -->
|
||||||
|
<g id="edge3" class="edge">
|
||||||
|
<title>nginx->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M201.11,-829C191.15,-829 182.88,-829 182.88,-829 182.88,-829 182.88,-622.1 182.88,-622.1"/>
|
||||||
|
<polygon fill="black" stroke="black" points="186.38,-622.1 182.88,-612.1 179.38,-622.1 186.38,-622.1"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="163" y="-737.91" font-family="Helvetica,sans-Serif" font-size="10.00">/graphql</text>
|
||||||
|
</g>
|
||||||
|
<!-- timeline -->
|
||||||
|
<g id="node5" class="node">
|
||||||
|
<title>timeline</title>
|
||||||
|
<path fill="none" stroke="black" d="M270,-741C270,-741 200,-741 200,-741 194,-741 188,-735 188,-729 188,-729 188,-693.25 188,-693.25 188,-687.25 194,-681.25 200,-681.25 200,-681.25 270,-681.25 270,-681.25 276,-681.25 282,-687.25 282,-693.25 282,-693.25 282,-729 282,-729 282,-735 276,-741 270,-741"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-723.7" font-family="Helvetica,sans-Serif" font-size="14.00">Timeline UI</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-706.45" font-family="Helvetica,sans-Serif" font-size="14.00">/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-689.2" font-family="Helvetica,sans-Serif" font-size="14.00">port 5173</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->timeline -->
|
||||||
|
<g id="edge4" class="edge">
|
||||||
|
<title>nginx->timeline</title>
|
||||||
|
<path fill="none" stroke="black" d="M235,-814.04C235,-814.04 235,-752.97 235,-752.97"/>
|
||||||
|
<polygon fill="black" stroke="black" points="238.5,-752.97 235,-742.97 231.5,-752.97 238.5,-752.97"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="233.5" y="-786.75" font-family="Helvetica,sans-Serif" font-size="10.00">/</text>
|
||||||
|
</g>
|
||||||
|
<!-- postgres -->
|
||||||
|
<g id="node6" class="node">
|
||||||
|
<title>postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M131.75,-434.03C131.75,-436.96 110.35,-439.34 84,-439.34 57.65,-439.34 36.25,-436.96 36.25,-434.03 36.25,-434.03 36.25,-386.22 36.25,-386.22 36.25,-383.29 57.65,-380.91 84,-380.91 110.35,-380.91 131.75,-383.29 131.75,-386.22 131.75,-386.22 131.75,-434.03 131.75,-434.03"/>
|
||||||
|
<path fill="none" stroke="black" d="M131.75,-434.03C131.75,-431.1 110.35,-428.72 84,-428.72 57.65,-428.72 36.25,-431.1 36.25,-434.03"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-414.07" font-family="Helvetica,sans-Serif" font-size="14.00">PostgreSQL</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-396.82" font-family="Helvetica,sans-Serif" font-size="14.00">port 5436</text>
|
||||||
|
</g>
|
||||||
|
<!-- django->postgres -->
|
||||||
|
<g id="edge6" class="edge">
|
||||||
|
<title>django->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M83,-680.89C83,-680.89 83,-450.97 83,-450.97"/>
|
||||||
|
<polygon fill="black" stroke="black" points="86.5,-450.97 83,-440.97 79.5,-450.97 86.5,-450.97"/>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->postgres -->
|
||||||
|
<g id="edge7" class="edge">
|
||||||
|
<title>fastapi->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M201.38,-550.41C201.38,-503.88 201.38,-420 201.38,-420 201.38,-420 143.59,-420 143.59,-420"/>
|
||||||
|
<polygon fill="black" stroke="black" points="143.59,-416.5 133.59,-420 143.59,-423.5 143.59,-416.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="266.38" y="-499.5" font-family="Helvetica,sans-Serif" font-size="10.00">read/write jobs</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->postgres -->
|
||||||
|
<g id="edge12" class="edge">
|
||||||
|
<title>fastapi->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M225,-550.39C225,-498.97 225,-400 225,-400 225,-400 143.64,-400 143.64,-400"/>
|
||||||
|
<polygon fill="black" stroke="black" points="143.64,-396.5 133.64,-400 143.64,-403.5 143.64,-396.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="125.25" y="-505.88" font-family="Helvetica,sans-Serif" font-size="10.00">callback updates</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="125.25" y="-493.12" font-family="Helvetica,sans-Serif" font-size="10.00">job status</text>
|
||||||
|
</g>
|
||||||
|
<!-- step_functions -->
|
||||||
|
<g id="node7" class="node">
|
||||||
|
<title>step_functions</title>
|
||||||
|
<path fill="none" stroke="black" d="M384.38,-440C384.38,-440 289.62,-440 289.62,-440 283.62,-440 277.62,-434 277.62,-428 277.62,-428 277.62,-392.25 277.62,-392.25 277.62,-386.25 283.62,-380.25 289.62,-380.25 289.62,-380.25 384.38,-380.25 384.38,-380.25 390.38,-380.25 396.38,-386.25 396.38,-392.25 396.38,-392.25 396.38,-428 396.38,-428 396.38,-434 390.38,-440 384.38,-440"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="337" y="-422.7" font-family="Helvetica,sans-Serif" font-size="14.00">Step Functions</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="337" y="-405.45" font-family="Helvetica,sans-Serif" font-size="14.00">Orchestration</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="337" y="-388.2" font-family="Helvetica,sans-Serif" font-size="14.00">state machine</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->step_functions -->
|
||||||
|
<g id="edge8" class="edge">
|
||||||
|
<title>fastapi->step_functions</title>
|
||||||
|
<path fill="none" stroke="black" d="M282.68,-581C289.69,-581 294.51,-581 294.51,-581 294.51,-581 294.51,-451.79 294.51,-451.79"/>
|
||||||
|
<polygon fill="black" stroke="black" points="298.01,-451.79 294.51,-441.79 291.01,-451.79 298.01,-451.79"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="407.25" y="-512.25" font-family="Helvetica,sans-Serif" font-size="10.00">boto3</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="407.25" y="-499.5" font-family="Helvetica,sans-Serif" font-size="10.00">start_execution()</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="407.25" y="-486.75" font-family="Helvetica,sans-Serif" font-size="10.00">execution_arn</text>
|
||||||
|
</g>
|
||||||
|
<!-- timeline->fastapi -->
|
||||||
|
<g id="edge5" class="edge">
|
||||||
|
<title>timeline->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M235,-680.86C235,-680.86 235,-622.24 235,-622.24"/>
|
||||||
|
<polygon fill="black" stroke="black" points="238.5,-622.24 235,-612.24 231.5,-622.24 238.5,-622.24"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="253" y="-642.75" font-family="Helvetica,sans-Serif" font-size="10.00">GraphQL</text>
|
||||||
|
</g>
|
||||||
|
<!-- lambda -->
|
||||||
|
<g id="node8" class="node">
|
||||||
|
<title>lambda</title>
|
||||||
|
<path fill="none" stroke="black" d="M486,-296.75C486,-296.75 368,-296.75 368,-296.75 362,-296.75 356,-290.75 356,-284.75 356,-284.75 356,-249 356,-249 356,-243 362,-237 368,-237 368,-237 486,-237 486,-237 492,-237 498,-243 498,-249 498,-249 498,-284.75 498,-284.75 498,-290.75 492,-296.75 486,-296.75"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="427" y="-279.45" font-family="Helvetica,sans-Serif" font-size="14.00">Lambda Function</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="427" y="-262.2" font-family="Helvetica,sans-Serif" font-size="14.00">FFmpeg container</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="427" y="-244.95" font-family="Helvetica,sans-Serif" font-size="14.00">transcoding</text>
|
||||||
|
</g>
|
||||||
|
<!-- step_functions->lambda -->
|
||||||
|
<g id="edge9" class="edge">
|
||||||
|
<title>step_functions->lambda</title>
|
||||||
|
<path fill="none" stroke="black" d="M376.19,-380.1C376.19,-380.1 376.19,-308.38 376.19,-308.38"/>
|
||||||
|
<polygon fill="black" stroke="black" points="379.69,-308.38 376.19,-298.38 372.69,-308.38 379.69,-308.38"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="407.12" y="-341.75" font-family="Helvetica,sans-Serif" font-size="10.00">invoke with</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="407.12" y="-329" font-family="Helvetica,sans-Serif" font-size="10.00">job parameters</text>
|
||||||
|
</g>
|
||||||
|
<!-- lambda->fastapi -->
|
||||||
|
<g id="edge11" class="edge">
|
||||||
|
<title>lambda->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M355.73,-267C306.05,-267 248.62,-267 248.62,-267 248.62,-267 248.62,-538.75 248.62,-538.75"/>
|
||||||
|
<polygon fill="black" stroke="black" points="245.13,-538.75 248.63,-548.75 252.13,-538.75 245.13,-538.75"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="571.62" y="-413.38" font-family="Helvetica,sans-Serif" font-size="10.00">POST /jobs/{id}/callback</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="571.62" y="-400.62" font-family="Helvetica,sans-Serif" font-size="10.00">update status</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3 -->
|
||||||
|
<g id="node9" class="node">
|
||||||
|
<title>s3</title>
|
||||||
|
<polygon fill="none" stroke="black" points="473.62,-153.5 470.62,-157.5 449.62,-157.5 446.62,-153.5 380.38,-153.5 380.38,-117.5 473.62,-117.5 473.62,-153.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="427" y="-130.82" font-family="Helvetica,sans-Serif" font-size="14.00">S3 Buckets</text>
|
||||||
|
</g>
|
||||||
|
<!-- lambda->s3 -->
|
||||||
|
<g id="edge10" class="edge">
|
||||||
|
<title>lambda->s3</title>
|
||||||
|
<path fill="none" stroke="black" d="M427,-236.73C427,-236.73 427,-165.27 427,-165.27"/>
|
||||||
|
<polygon fill="black" stroke="black" points="430.5,-165.27 427,-155.27 423.5,-165.27 430.5,-165.27"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="464.5" y="-198.5" font-family="Helvetica,sans-Serif" font-size="10.00">download input</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="464.5" y="-185.75" font-family="Helvetica,sans-Serif" font-size="10.00">upload output</text>
|
||||||
|
</g>
|
||||||
|
<!-- bucket_in -->
|
||||||
|
<g id="node10" class="node">
|
||||||
|
<title>bucket_in</title>
|
||||||
|
<polygon fill="none" stroke="black" points="379.75,-58.5 272.25,-58.5 272.25,-16 385.75,-16 385.75,-52.5 379.75,-58.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="379.75,-58.5 379.75,-52.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="385.75,-52.5 379.75,-52.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="329" y="-41.2" font-family="Helvetica,sans-Serif" font-size="14.00">mpr-media-in/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="329" y="-23.95" font-family="Helvetica,sans-Serif" font-size="14.00">input videos</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3->bucket_in -->
|
||||||
|
<g id="edge13" class="edge">
|
||||||
|
<title>s3->bucket_in</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M380.09,-136C373.1,-136 368.19,-136 368.19,-136 368.19,-136 368.19,-87.72 368.19,-58.68"/>
|
||||||
|
</g>
|
||||||
|
<!-- bucket_out -->
|
||||||
|
<g id="node11" class="node">
|
||||||
|
<title>bucket_out</title>
|
||||||
|
<polygon fill="none" stroke="black" points="582.12,-58.5 443.88,-58.5 443.88,-16 588.12,-16 588.12,-52.5 582.12,-58.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="582.12,-58.5 582.12,-52.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="588.12,-52.5 582.12,-52.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="516" y="-41.2" font-family="Helvetica,sans-Serif" font-size="14.00">mpr-media-out/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="516" y="-23.95" font-family="Helvetica,sans-Serif" font-size="14.00">transcoded output</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3->bucket_out -->
|
||||||
|
<g id="edge14" class="edge">
|
||||||
|
<title>s3->bucket_out</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M458.75,-117.02C458.75,-100.45 458.75,-76.15 458.75,-58.73"/>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 16 KiB |
83
docs/architecture/01c-gcp-architecture.dot
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
digraph gcp_architecture {
|
||||||
|
rankdir=TB
|
||||||
|
node [shape=box, style=rounded, fontname="Helvetica"]
|
||||||
|
edge [fontname="Helvetica", fontsize=10]
|
||||||
|
|
||||||
|
labelloc="t"
|
||||||
|
label="MPR - GCP Architecture (Cloud Run Jobs + GCS)"
|
||||||
|
fontsize=16
|
||||||
|
fontname="Helvetica-Bold"
|
||||||
|
|
||||||
|
graph [splines=ortho, nodesep=0.8, ranksep=0.8]
|
||||||
|
|
||||||
|
// External
|
||||||
|
subgraph cluster_external {
|
||||||
|
label="External"
|
||||||
|
style=dashed
|
||||||
|
color=gray
|
||||||
|
|
||||||
|
browser [label="Browser\nmpr.mcrn.ar", shape=ellipse]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nginx reverse proxy
|
||||||
|
subgraph cluster_proxy {
|
||||||
|
label="Reverse Proxy"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#e8f4f8"
|
||||||
|
|
||||||
|
nginx [label="nginx\nport 80"]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Application layer
|
||||||
|
subgraph cluster_apps {
|
||||||
|
label="Application Layer"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#f0f8e8"
|
||||||
|
|
||||||
|
django [label="Django Admin\n/admin\nport 8701"]
|
||||||
|
fastapi [label="GraphQL API\n/graphql\nport 8702"]
|
||||||
|
timeline [label="Timeline UI\n/\nport 5173"]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Data layer (still local)
|
||||||
|
subgraph cluster_data {
|
||||||
|
label="Data Layer"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#f8e8f0"
|
||||||
|
|
||||||
|
postgres [label="PostgreSQL\nport 5436", shape=cylinder]
|
||||||
|
}
|
||||||
|
|
||||||
|
// GCP layer
|
||||||
|
subgraph cluster_gcp {
|
||||||
|
label="Google Cloud"
|
||||||
|
style=filled
|
||||||
|
fillcolor="#e8f0fd"
|
||||||
|
|
||||||
|
cloud_run_job [label="Cloud Run Job\nFFmpeg container\ntranscoding"]
|
||||||
|
gcs [label="GCS Buckets\n(S3-compat API)", shape=folder]
|
||||||
|
bucket_in [label="mpr-media-in/\ninput videos", shape=note]
|
||||||
|
bucket_out [label="mpr-media-out/\ntranscoded output", shape=note]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connections
|
||||||
|
browser -> nginx [label="HTTP"]
|
||||||
|
|
||||||
|
nginx -> django [xlabel="/admin"]
|
||||||
|
nginx -> fastapi [xlabel="/graphql"]
|
||||||
|
nginx -> timeline [xlabel="/"]
|
||||||
|
|
||||||
|
timeline -> fastapi [label="GraphQL"]
|
||||||
|
django -> postgres
|
||||||
|
|
||||||
|
fastapi -> postgres [label="read/write jobs"]
|
||||||
|
fastapi -> cloud_run_job [label="google-cloud-run\nrun_job() + payload\nexecution_name"]
|
||||||
|
|
||||||
|
cloud_run_job -> gcs [label="S3 compat (HMAC)\ndownload input\nupload output"]
|
||||||
|
cloud_run_job -> fastapi [label="POST /jobs/{id}/callback\nupdate status"]
|
||||||
|
|
||||||
|
fastapi -> postgres [label="callback updates\njob status"]
|
||||||
|
|
||||||
|
gcs -> bucket_in [style=dotted, arrowhead=none]
|
||||||
|
gcs -> bucket_out [style=dotted, arrowhead=none]
|
||||||
|
}
|
||||||
210
docs/architecture/01c-gcp-architecture.svg
Normal file
@@ -0,0 +1,210 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
|
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||||
|
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||||
|
<!-- Generated by graphviz version 14.1.2 (0)
|
||||||
|
-->
|
||||||
|
<!-- Title: gcp_architecture Pages: 1 -->
|
||||||
|
<svg width="653pt" height="957pt"
|
||||||
|
viewBox="0.00 0.00 653.00 957.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||||
|
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 953.35)">
|
||||||
|
<title>gcp_architecture</title>
|
||||||
|
<polygon fill="white" stroke="none" points="-4,4 -4,-953.35 649.25,-953.35 649.25,4 -4,4"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="322.62" y="-930.15" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - GCP Architecture (Cloud Run Jobs + GCS)</text>
|
||||||
|
<g id="clust1" class="cluster">
|
||||||
|
<title>cluster_external</title>
|
||||||
|
<polygon fill="none" stroke="gray" stroke-dasharray="5,2" points="155,-810.25 155,-913.85 315,-913.85 315,-810.25 155,-810.25"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-894.65" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">External</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust2" class="cluster">
|
||||||
|
<title>cluster_proxy</title>
|
||||||
|
<polygon fill="#e8f4f8" stroke="black" points="162,-682.5 162,-768.5 308,-768.5 308,-682.5 162,-682.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-749.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Reverse Proxy</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust3" class="cluster">
|
||||||
|
<title>cluster_apps</title>
|
||||||
|
<polygon fill="#f0f8e8" stroke="black" points="8,-418.75 8,-652.5 290,-652.5 290,-418.75 8,-418.75"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="149" y="-633.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Application Layer</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust4" class="cluster">
|
||||||
|
<title>cluster_data</title>
|
||||||
|
<polygon fill="#f8e8f0" stroke="black" points="27,-248.91 27,-350.84 141,-350.84 141,-248.91 27,-248.91"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-331.64" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Data Layer</text>
|
||||||
|
</g>
|
||||||
|
<g id="clust5" class="cluster">
|
||||||
|
<title>cluster_gcp</title>
|
||||||
|
<polygon fill="#e8f0fd" stroke="black" points="299,-8 299,-351.5 631,-351.5 631,-8 299,-8"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="465" y="-332.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Google Cloud</text>
|
||||||
|
</g>
|
||||||
|
<!-- browser -->
|
||||||
|
<g id="node1" class="node">
|
||||||
|
<title>browser</title>
|
||||||
|
<ellipse fill="none" stroke="black" cx="235" cy="-848.3" rx="71.77" ry="30.05"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-852.25" font-family="Helvetica,sans-Serif" font-size="14.00">Browser</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-835" font-family="Helvetica,sans-Serif" font-size="14.00">mpr.mcrn.ar</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx -->
|
||||||
|
<g id="node2" class="node">
|
||||||
|
<title>nginx</title>
|
||||||
|
<path fill="none" stroke="black" d="M256.5,-733C256.5,-733 213.5,-733 213.5,-733 207.5,-733 201.5,-727 201.5,-721 201.5,-721 201.5,-702.5 201.5,-702.5 201.5,-696.5 207.5,-690.5 213.5,-690.5 213.5,-690.5 256.5,-690.5 256.5,-690.5 262.5,-690.5 268.5,-696.5 268.5,-702.5 268.5,-702.5 268.5,-721 268.5,-721 268.5,-727 262.5,-733 256.5,-733"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-715.7" font-family="Helvetica,sans-Serif" font-size="14.00">nginx</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-698.45" font-family="Helvetica,sans-Serif" font-size="14.00">port 80</text>
|
||||||
|
</g>
|
||||||
|
<!-- browser->nginx -->
|
||||||
|
<g id="edge1" class="edge">
|
||||||
|
<title>browser->nginx</title>
|
||||||
|
<path fill="none" stroke="black" d="M235,-818C235,-818 235,-745 235,-745"/>
|
||||||
|
<polygon fill="black" stroke="black" points="238.5,-745 235,-735 231.5,-745 238.5,-745"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="247.75" y="-779.75" font-family="Helvetica,sans-Serif" font-size="10.00">HTTP</text>
|
||||||
|
</g>
|
||||||
|
<!-- django -->
|
||||||
|
<g id="node3" class="node">
|
||||||
|
<title>django</title>
|
||||||
|
<path fill="none" stroke="black" d="M117.75,-617C117.75,-617 28.25,-617 28.25,-617 22.25,-617 16.25,-611 16.25,-605 16.25,-605 16.25,-569.25 16.25,-569.25 16.25,-563.25 22.25,-557.25 28.25,-557.25 28.25,-557.25 117.75,-557.25 117.75,-557.25 123.75,-557.25 129.75,-563.25 129.75,-569.25 129.75,-569.25 129.75,-605 129.75,-605 129.75,-611 123.75,-617 117.75,-617"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="73" y="-599.7" font-family="Helvetica,sans-Serif" font-size="14.00">Django Admin</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="73" y="-582.45" font-family="Helvetica,sans-Serif" font-size="14.00">/admin</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="73" y="-565.2" font-family="Helvetica,sans-Serif" font-size="14.00">port 8701</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->django -->
|
||||||
|
<g id="edge2" class="edge">
|
||||||
|
<title>nginx->django</title>
|
||||||
|
<path fill="none" stroke="black" d="M201.04,-719C153.54,-719 73,-719 73,-719 73,-719 73,-628.89 73,-628.89"/>
|
||||||
|
<polygon fill="black" stroke="black" points="76.5,-628.89 73,-618.89 69.5,-628.89 76.5,-628.89"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="75.09" y="-722.25" font-family="Helvetica,sans-Serif" font-size="10.00">/admin</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi -->
|
||||||
|
<g id="node4" class="node">
|
||||||
|
<title>fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M270.25,-486.5C270.25,-486.5 189.75,-486.5 189.75,-486.5 183.75,-486.5 177.75,-480.5 177.75,-474.5 177.75,-474.5 177.75,-438.75 177.75,-438.75 177.75,-432.75 183.75,-426.75 189.75,-426.75 189.75,-426.75 270.25,-426.75 270.25,-426.75 276.25,-426.75 282.25,-432.75 282.25,-438.75 282.25,-438.75 282.25,-474.5 282.25,-474.5 282.25,-480.5 276.25,-486.5 270.25,-486.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="230" y="-469.2" font-family="Helvetica,sans-Serif" font-size="14.00">GraphQL API</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="230" y="-451.95" font-family="Helvetica,sans-Serif" font-size="14.00">/graphql</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="230" y="-434.7" font-family="Helvetica,sans-Serif" font-size="14.00">port 8702</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->fastapi -->
|
||||||
|
<g id="edge3" class="edge">
|
||||||
|
<title>nginx->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M201.11,-705C191.15,-705 182.88,-705 182.88,-705 182.88,-705 182.88,-498.1 182.88,-498.1"/>
|
||||||
|
<polygon fill="black" stroke="black" points="186.38,-498.1 182.88,-488.1 179.38,-498.1 186.38,-498.1"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="163" y="-613.91" font-family="Helvetica,sans-Serif" font-size="10.00">/graphql</text>
|
||||||
|
</g>
|
||||||
|
<!-- timeline -->
|
||||||
|
<g id="node5" class="node">
|
||||||
|
<title>timeline</title>
|
||||||
|
<path fill="none" stroke="black" d="M270,-617C270,-617 200,-617 200,-617 194,-617 188,-611 188,-605 188,-605 188,-569.25 188,-569.25 188,-563.25 194,-557.25 200,-557.25 200,-557.25 270,-557.25 270,-557.25 276,-557.25 282,-563.25 282,-569.25 282,-569.25 282,-605 282,-605 282,-611 276,-617 270,-617"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-599.7" font-family="Helvetica,sans-Serif" font-size="14.00">Timeline UI</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-582.45" font-family="Helvetica,sans-Serif" font-size="14.00">/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="235" y="-565.2" font-family="Helvetica,sans-Serif" font-size="14.00">port 5173</text>
|
||||||
|
</g>
|
||||||
|
<!-- nginx->timeline -->
|
||||||
|
<g id="edge4" class="edge">
|
||||||
|
<title>nginx->timeline</title>
|
||||||
|
<path fill="none" stroke="black" d="M235,-690.04C235,-690.04 235,-628.97 235,-628.97"/>
|
||||||
|
<polygon fill="black" stroke="black" points="238.5,-628.97 235,-618.97 231.5,-628.97 238.5,-628.97"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="233.5" y="-662.75" font-family="Helvetica,sans-Serif" font-size="10.00">/</text>
|
||||||
|
</g>
|
||||||
|
<!-- postgres -->
|
||||||
|
<g id="node6" class="node">
|
||||||
|
<title>postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M131.75,-310.03C131.75,-312.96 110.35,-315.34 84,-315.34 57.65,-315.34 36.25,-312.96 36.25,-310.03 36.25,-310.03 36.25,-262.22 36.25,-262.22 36.25,-259.29 57.65,-256.91 84,-256.91 110.35,-256.91 131.75,-259.29 131.75,-262.22 131.75,-262.22 131.75,-310.03 131.75,-310.03"/>
|
||||||
|
<path fill="none" stroke="black" d="M131.75,-310.03C131.75,-307.1 110.35,-304.72 84,-304.72 57.65,-304.72 36.25,-307.1 36.25,-310.03"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-290.07" font-family="Helvetica,sans-Serif" font-size="14.00">PostgreSQL</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="84" y="-272.82" font-family="Helvetica,sans-Serif" font-size="14.00">port 5436</text>
|
||||||
|
</g>
|
||||||
|
<!-- django->postgres -->
|
||||||
|
<g id="edge6" class="edge">
|
||||||
|
<title>django->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M59.62,-556.89C59.62,-556.89 59.62,-326.97 59.62,-326.97"/>
|
||||||
|
<polygon fill="black" stroke="black" points="63.13,-326.97 59.63,-316.97 56.13,-326.97 63.13,-326.97"/>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->postgres -->
|
||||||
|
<g id="edge7" class="edge">
|
||||||
|
<title>fastapi->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M177.34,-467C135.16,-467 83,-467 83,-467 83,-467 83,-327.1 83,-327.1"/>
|
||||||
|
<polygon fill="black" stroke="black" points="86.5,-327.1 83,-317.1 79.5,-327.1 86.5,-327.1"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="266.38" y="-375.5" font-family="Helvetica,sans-Serif" font-size="10.00">read/write jobs</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->postgres -->
|
||||||
|
<g id="edge11" class="edge">
|
||||||
|
<title>fastapi->postgres</title>
|
||||||
|
<path fill="none" stroke="black" d="M177.57,-447C143.88,-447 106.38,-447 106.38,-447 106.38,-447 106.38,-327.15 106.38,-327.15"/>
|
||||||
|
<polygon fill="black" stroke="black" points="109.88,-327.15 106.38,-317.15 102.88,-327.15 109.88,-327.15"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="125.25" y="-381.88" font-family="Helvetica,sans-Serif" font-size="10.00">callback updates</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="125.25" y="-369.12" font-family="Helvetica,sans-Serif" font-size="10.00">job status</text>
|
||||||
|
</g>
|
||||||
|
<!-- cloud_run_job -->
|
||||||
|
<g id="node7" class="node">
|
||||||
|
<title>cloud_run_job</title>
|
||||||
|
<path fill="none" stroke="black" d="M505,-316C505,-316 387,-316 387,-316 381,-316 375,-310 375,-304 375,-304 375,-268.25 375,-268.25 375,-262.25 381,-256.25 387,-256.25 387,-256.25 505,-256.25 505,-256.25 511,-256.25 517,-262.25 517,-268.25 517,-268.25 517,-304 517,-304 517,-310 511,-316 505,-316"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="446" y="-298.7" font-family="Helvetica,sans-Serif" font-size="14.00">Cloud Run Job</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="446" y="-281.45" font-family="Helvetica,sans-Serif" font-size="14.00">FFmpeg container</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="446" y="-264.2" font-family="Helvetica,sans-Serif" font-size="14.00">transcoding</text>
|
||||||
|
</g>
|
||||||
|
<!-- fastapi->cloud_run_job -->
|
||||||
|
<g id="edge8" class="edge">
|
||||||
|
<title>fastapi->cloud_run_job</title>
|
||||||
|
<path fill="none" stroke="black" d="M247.42,-426.41C247.42,-379.88 247.42,-296 247.42,-296 247.42,-296 363.07,-296 363.07,-296"/>
|
||||||
|
<polygon fill="black" stroke="black" points="363.07,-299.5 373.07,-296 363.07,-292.5 363.07,-299.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="414.38" y="-388.25" font-family="Helvetica,sans-Serif" font-size="10.00">google-cloud-run</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="414.38" y="-375.5" font-family="Helvetica,sans-Serif" font-size="10.00">run_job() + payload</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="414.38" y="-362.75" font-family="Helvetica,sans-Serif" font-size="10.00">execution_name</text>
|
||||||
|
</g>
|
||||||
|
<!-- timeline->fastapi -->
|
||||||
|
<g id="edge5" class="edge">
|
||||||
|
<title>timeline->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M235,-556.86C235,-556.86 235,-498.24 235,-498.24"/>
|
||||||
|
<polygon fill="black" stroke="black" points="238.5,-498.24 235,-488.24 231.5,-498.24 238.5,-498.24"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="253" y="-518.75" font-family="Helvetica,sans-Serif" font-size="10.00">GraphQL</text>
|
||||||
|
</g>
|
||||||
|
<!-- cloud_run_job->fastapi -->
|
||||||
|
<g id="edge10" class="edge">
|
||||||
|
<title>cloud_run_job->fastapi</title>
|
||||||
|
<path fill="none" stroke="black" d="M374.7,-276C306.06,-276 212.58,-276 212.58,-276 212.58,-276 212.58,-414.88 212.58,-414.88"/>
|
||||||
|
<polygon fill="black" stroke="black" points="209.08,-414.88 212.58,-424.88 216.08,-414.88 209.08,-414.88"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="585.62" y="-381.88" font-family="Helvetica,sans-Serif" font-size="10.00">POST /jobs/{id}/callback</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="585.62" y="-369.12" font-family="Helvetica,sans-Serif" font-size="10.00">update status</text>
|
||||||
|
</g>
|
||||||
|
<!-- gcs -->
|
||||||
|
<g id="node8" class="node">
|
||||||
|
<title>gcs</title>
|
||||||
|
<polygon fill="none" stroke="black" points="510.25,-160 507.25,-164 486.25,-164 483.25,-160 381.75,-160 381.75,-117.5 510.25,-117.5 510.25,-160"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="446" y="-142.7" font-family="Helvetica,sans-Serif" font-size="14.00">GCS Buckets</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="446" y="-125.45" font-family="Helvetica,sans-Serif" font-size="14.00">(S3-compat API)</text>
|
||||||
|
</g>
|
||||||
|
<!-- cloud_run_job->gcs -->
|
||||||
|
<g id="edge9" class="edge">
|
||||||
|
<title>cloud_run_job->gcs</title>
|
||||||
|
<path fill="none" stroke="black" d="M446,-255.95C446,-255.95 446,-171.81 446,-171.81"/>
|
||||||
|
<polygon fill="black" stroke="black" points="449.5,-171.81 446,-161.81 442.5,-171.81 449.5,-171.81"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="492.12" y="-217.75" font-family="Helvetica,sans-Serif" font-size="10.00">S3 compat (HMAC)</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="492.12" y="-205" font-family="Helvetica,sans-Serif" font-size="10.00">download input</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="492.12" y="-192.25" font-family="Helvetica,sans-Serif" font-size="10.00">upload output</text>
|
||||||
|
</g>
|
||||||
|
<!-- bucket_in -->
|
||||||
|
<g id="node9" class="node">
|
||||||
|
<title>bucket_in</title>
|
||||||
|
<polygon fill="none" stroke="black" points="414.75,-58.5 307.25,-58.5 307.25,-16 420.75,-16 420.75,-52.5 414.75,-58.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="414.75,-58.5 414.75,-52.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="420.75,-52.5 414.75,-52.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="364" y="-41.2" font-family="Helvetica,sans-Serif" font-size="14.00">mpr-media-in/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="364" y="-23.95" font-family="Helvetica,sans-Serif" font-size="14.00">input videos</text>
|
||||||
|
</g>
|
||||||
|
<!-- gcs->bucket_in -->
|
||||||
|
<g id="edge12" class="edge">
|
||||||
|
<title>gcs->bucket_in</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M401.25,-117.22C401.25,-100 401.25,-75.96 401.25,-58.74"/>
|
||||||
|
</g>
|
||||||
|
<!-- bucket_out -->
|
||||||
|
<g id="node10" class="node">
|
||||||
|
<title>bucket_out</title>
|
||||||
|
<polygon fill="none" stroke="black" points="617.12,-58.5 478.88,-58.5 478.88,-16 623.12,-16 623.12,-52.5 617.12,-58.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="617.12,-58.5 617.12,-52.5"/>
|
||||||
|
<polyline fill="none" stroke="black" points="623.12,-52.5 617.12,-52.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="551" y="-41.2" font-family="Helvetica,sans-Serif" font-size="14.00">mpr-media-out/</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="551" y="-23.95" font-family="Helvetica,sans-Serif" font-size="14.00">transcoded output</text>
|
||||||
|
</g>
|
||||||
|
<!-- gcs->bucket_out -->
|
||||||
|
<g id="edge13" class="edge">
|
||||||
|
<title>gcs->bucket_out</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M494.56,-117.22C494.56,-100 494.56,-75.96 494.56,-58.74"/>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 15 KiB |
@@ -10,13 +10,13 @@ digraph data_model {
|
|||||||
|
|
||||||
graph [splines=ortho, nodesep=0.6, ranksep=1.2]
|
graph [splines=ortho, nodesep=0.6, ranksep=1.2]
|
||||||
|
|
||||||
MediaAsset [label="{MediaAsset|id: UUID (PK)\lfilename: str\lfile_path: str\lfile_size: int?\lstatus: pending/ready/error\lerror_message: str?\l|duration: float?\lvideo_codec: str?\laudio_codec: str?\lwidth: int?\lheight: int?\lframerate: float?\lbitrate: int?\lproperties: JSON\l|comments: str\ltags: JSON[]\l|created_at: datetime\lupdated_at: datetime\l}"]
|
MediaAsset [label="{MediaAsset|id: UUID (PK)\lfilename: str\lfile_path: str (S3 key)\lfile_size: int?\lstatus: pending/ready/error\lerror_message: str?\l|duration: float?\lvideo_codec: str?\laudio_codec: str?\lwidth: int?\lheight: int?\lframerate: float?\lbitrate: int?\lproperties: JSON\l|comments: str\ltags: JSON[]\l|created_at: datetime\lupdated_at: datetime\l}"]
|
||||||
|
|
||||||
TranscodePreset [label="{TranscodePreset|id: UUID (PK)\lname: str (unique)\ldescription: str\lis_builtin: bool\l|container: str\l|video_codec: str\lvideo_bitrate: str?\lvideo_crf: int?\lvideo_preset: str?\lresolution: str?\lframerate: float?\l|audio_codec: str\laudio_bitrate: str?\laudio_channels: int?\laudio_samplerate: int?\l|extra_args: JSON[]\l|created_at: datetime\lupdated_at: datetime\l}"]
|
TranscodePreset [label="{TranscodePreset|id: UUID (PK)\lname: str (unique)\ldescription: str\lis_builtin: bool\l|container: str\l|video_codec: str\lvideo_bitrate: str?\lvideo_crf: int?\lvideo_preset: str?\lresolution: str?\lframerate: float?\l|audio_codec: str\laudio_bitrate: str?\laudio_channels: int?\laudio_samplerate: int?\l|extra_args: JSON[]\l|created_at: datetime\lupdated_at: datetime\l}"]
|
||||||
|
|
||||||
TranscodeJob [label="{TranscodeJob|id: UUID (PK)\l|source_asset_id: UUID (FK)\l|preset_id: UUID? (FK)\lpreset_snapshot: JSON\l|trim_start: float?\ltrim_end: float?\l|output_filename: str\loutput_path: str?\loutput_asset_id: UUID? (FK)\l|status: pending/processing/...\lprogress: float (0-100)\lcurrent_frame: int?\lcurrent_time: float?\lspeed: str?\lerror_message: str?\l|celery_task_id: str?\lpriority: int\l|created_at: datetime\lstarted_at: datetime?\lcompleted_at: datetime?\l}"]
|
TranscodeJob [label="{TranscodeJob|id: UUID (PK)\l|source_asset_id: UUID (FK)\l|preset_id: UUID? (FK)\lpreset_snapshot: JSON\l|trim_start: float?\ltrim_end: float?\l|output_filename: str\loutput_path: str? (S3 key)\loutput_asset_id: UUID? (FK)\l|status: pending/processing/...\lprogress: float (0-100)\lcurrent_frame: int?\lcurrent_time: float?\lspeed: str?\lerror_message: str?\l|celery_task_id: str?\lexecution_arn: str?\lpriority: int\l|created_at: datetime\lstarted_at: datetime?\lcompleted_at: datetime?\l}"]
|
||||||
|
|
||||||
MediaAsset -> TranscodeJob [label="1:N source_asset"]
|
MediaAsset -> TranscodeJob [xlabel="1:N source_asset"]
|
||||||
TranscodePreset -> TranscodeJob [label="1:N preset"]
|
TranscodePreset -> TranscodeJob [xlabel="1:N preset"]
|
||||||
TranscodeJob -> MediaAsset [label="1:1 output_asset", style=dashed]
|
TranscodeJob -> MediaAsset [xlabel="1:1 output_asset", style=dashed]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||||
<!-- Generated by graphviz version 14.1.1 (0)
|
<!-- Generated by graphviz version 14.1.2 (0)
|
||||||
-->
|
-->
|
||||||
<!-- Title: data_model Pages: 1 -->
|
<!-- Title: data_model Pages: 1 -->
|
||||||
<svg width="2218pt" height="286pt"
|
<svg width="2134pt" height="286pt"
|
||||||
viewBox="0.00 0.00 2218.00 286.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
viewBox="0.00 0.00 2134.00 286.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 282)">
|
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 282)">
|
||||||
<title>data_model</title>
|
<title>data_model</title>
|
||||||
<polygon fill="white" stroke="none" points="-4,4 -4,-282 2213.5,-282 2213.5,4 -4,4"/>
|
<polygon fill="white" stroke="none" points="-4,4 -4,-282 2130.25,-282 2130.25,4 -4,4"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1104.75" y="-258.8" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - Data Model</text>
|
<text xml:space="preserve" text-anchor="middle" x="1063.12" y="-258.8" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - Data Model</text>
|
||||||
<!-- MediaAsset -->
|
<!-- MediaAsset -->
|
||||||
<g id="node1" class="node">
|
<g id="node1" class="node">
|
||||||
<title>MediaAsset</title>
|
<title>MediaAsset</title>
|
||||||
@@ -18,7 +18,7 @@
|
|||||||
<polyline fill="none" stroke="black" points="197.75,-134 197.75,-250"/>
|
<polyline fill="none" stroke="black" points="197.75,-134 197.75,-250"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="205.75" y="-222.05" font-family="Helvetica,sans-Serif" font-size="11.00">id: UUID (PK)</text>
|
<text xml:space="preserve" text-anchor="start" x="205.75" y="-222.05" font-family="Helvetica,sans-Serif" font-size="11.00">id: UUID (PK)</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="205.75" y="-208.55" font-family="Helvetica,sans-Serif" font-size="11.00">filename: str</text>
|
<text xml:space="preserve" text-anchor="start" x="205.75" y="-208.55" font-family="Helvetica,sans-Serif" font-size="11.00">filename: str</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="205.75" y="-195.05" font-family="Helvetica,sans-Serif" font-size="11.00">file_path: str</text>
|
<text xml:space="preserve" text-anchor="start" x="205.75" y="-195.05" font-family="Helvetica,sans-Serif" font-size="11.00">file_path: str (S3 key)</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="205.75" y="-181.55" font-family="Helvetica,sans-Serif" font-size="11.00">file_size: int?</text>
|
<text xml:space="preserve" text-anchor="start" x="205.75" y="-181.55" font-family="Helvetica,sans-Serif" font-size="11.00">file_size: int?</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="205.75" y="-168.05" font-family="Helvetica,sans-Serif" font-size="11.00">status: pending/ready/error</text>
|
<text xml:space="preserve" text-anchor="start" x="205.75" y="-168.05" font-family="Helvetica,sans-Serif" font-size="11.00">status: pending/ready/error</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="205.75" y="-154.55" font-family="Helvetica,sans-Serif" font-size="11.00">error_message: str?</text>
|
<text xml:space="preserve" text-anchor="start" x="205.75" y="-154.55" font-family="Helvetica,sans-Serif" font-size="11.00">error_message: str?</text>
|
||||||
@@ -41,43 +41,44 @@
|
|||||||
<!-- TranscodeJob -->
|
<!-- TranscodeJob -->
|
||||||
<g id="node3" class="node">
|
<g id="node3" class="node">
|
||||||
<title>TranscodeJob</title>
|
<title>TranscodeJob</title>
|
||||||
<polygon fill="none" stroke="black" points="995.25,-86.5 995.25,-175.5 2209.5,-175.5 2209.5,-86.5 995.25,-86.5"/>
|
<polygon fill="none" stroke="black" points="912,-147.5 912,-236.5 2126.25,-236.5 2126.25,-147.5 912,-147.5"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1039.25" y="-127.3" font-family="Helvetica,sans-Serif" font-size="11.00">TranscodeJob</text>
|
<text xml:space="preserve" text-anchor="middle" x="956" y="-188.3" font-family="Helvetica,sans-Serif" font-size="11.00">TranscodeJob</text>
|
||||||
<polyline fill="none" stroke="black" points="1083.25,-86.5 1083.25,-175.5"/>
|
<polyline fill="none" stroke="black" points="1000,-147.5 1000,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1091.25" y="-127.3" font-family="Helvetica,sans-Serif" font-size="11.00">id: UUID (PK)</text>
|
<text xml:space="preserve" text-anchor="start" x="1008" y="-188.3" font-family="Helvetica,sans-Serif" font-size="11.00">id: UUID (PK)</text>
|
||||||
<polyline fill="none" stroke="black" points="1171.25,-86.5 1171.25,-175.5"/>
|
<polyline fill="none" stroke="black" points="1088,-147.5 1088,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1179.25" y="-127.3" font-family="Helvetica,sans-Serif" font-size="11.00">source_asset_id: UUID (FK)</text>
|
<text xml:space="preserve" text-anchor="start" x="1096" y="-188.3" font-family="Helvetica,sans-Serif" font-size="11.00">source_asset_id: UUID (FK)</text>
|
||||||
<polyline fill="none" stroke="black" points="1335.75,-86.5 1335.75,-175.5"/>
|
<polyline fill="none" stroke="black" points="1252.5,-147.5 1252.5,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1343.75" y="-134.05" font-family="Helvetica,sans-Serif" font-size="11.00">preset_id: UUID? (FK)</text>
|
<text xml:space="preserve" text-anchor="start" x="1260.5" y="-195.05" font-family="Helvetica,sans-Serif" font-size="11.00">preset_id: UUID? (FK)</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1343.75" y="-120.55" font-family="Helvetica,sans-Serif" font-size="11.00">preset_snapshot: JSON</text>
|
<text xml:space="preserve" text-anchor="start" x="1260.5" y="-181.55" font-family="Helvetica,sans-Serif" font-size="11.00">preset_snapshot: JSON</text>
|
||||||
<polyline fill="none" stroke="black" points="1477,-86.5 1477,-175.5"/>
|
<polyline fill="none" stroke="black" points="1393.75,-147.5 1393.75,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1485" y="-134.05" font-family="Helvetica,sans-Serif" font-size="11.00">trim_start: float?</text>
|
<text xml:space="preserve" text-anchor="start" x="1401.75" y="-195.05" font-family="Helvetica,sans-Serif" font-size="11.00">trim_start: float?</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1485" y="-120.55" font-family="Helvetica,sans-Serif" font-size="11.00">trim_end: float?</text>
|
<text xml:space="preserve" text-anchor="start" x="1401.75" y="-181.55" font-family="Helvetica,sans-Serif" font-size="11.00">trim_end: float?</text>
|
||||||
<polyline fill="none" stroke="black" points="1585.25,-86.5 1585.25,-175.5"/>
|
<polyline fill="none" stroke="black" points="1502,-147.5 1502,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1593.25" y="-140.8" font-family="Helvetica,sans-Serif" font-size="11.00">output_filename: str</text>
|
<text xml:space="preserve" text-anchor="start" x="1510" y="-201.8" font-family="Helvetica,sans-Serif" font-size="11.00">output_filename: str</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1593.25" y="-127.3" font-family="Helvetica,sans-Serif" font-size="11.00">output_path: str?</text>
|
<text xml:space="preserve" text-anchor="start" x="1510" y="-188.3" font-family="Helvetica,sans-Serif" font-size="11.00">output_path: str? (S3 key)</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1593.25" y="-113.8" font-family="Helvetica,sans-Serif" font-size="11.00">output_asset_id: UUID? (FK)</text>
|
<text xml:space="preserve" text-anchor="start" x="1510" y="-174.8" font-family="Helvetica,sans-Serif" font-size="11.00">output_asset_id: UUID? (FK)</text>
|
||||||
<polyline fill="none" stroke="black" points="1755,-86.5 1755,-175.5"/>
|
<polyline fill="none" stroke="black" points="1671.75,-147.5 1671.75,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1763" y="-161.05" font-family="Helvetica,sans-Serif" font-size="11.00">status: pending/processing/...</text>
|
<text xml:space="preserve" text-anchor="start" x="1679.75" y="-222.05" font-family="Helvetica,sans-Serif" font-size="11.00">status: pending/processing/...</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1763" y="-147.55" font-family="Helvetica,sans-Serif" font-size="11.00">progress: float (0-100)</text>
|
<text xml:space="preserve" text-anchor="start" x="1679.75" y="-208.55" font-family="Helvetica,sans-Serif" font-size="11.00">progress: float (0-100)</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1763" y="-134.05" font-family="Helvetica,sans-Serif" font-size="11.00">current_frame: int?</text>
|
<text xml:space="preserve" text-anchor="start" x="1679.75" y="-195.05" font-family="Helvetica,sans-Serif" font-size="11.00">current_frame: int?</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1763" y="-120.55" font-family="Helvetica,sans-Serif" font-size="11.00">current_time: float?</text>
|
<text xml:space="preserve" text-anchor="start" x="1679.75" y="-181.55" font-family="Helvetica,sans-Serif" font-size="11.00">current_time: float?</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1763" y="-107.05" font-family="Helvetica,sans-Serif" font-size="11.00">speed: str?</text>
|
<text xml:space="preserve" text-anchor="start" x="1679.75" y="-168.05" font-family="Helvetica,sans-Serif" font-size="11.00">speed: str?</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1763" y="-93.55" font-family="Helvetica,sans-Serif" font-size="11.00">error_message: str?</text>
|
<text xml:space="preserve" text-anchor="start" x="1679.75" y="-154.55" font-family="Helvetica,sans-Serif" font-size="11.00">error_message: str?</text>
|
||||||
<polyline fill="none" stroke="black" points="1934.5,-86.5 1934.5,-175.5"/>
|
<polyline fill="none" stroke="black" points="1851.25,-147.5 1851.25,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1942.5" y="-134.05" font-family="Helvetica,sans-Serif" font-size="11.00">celery_task_id: str?</text>
|
<text xml:space="preserve" text-anchor="start" x="1859.25" y="-201.8" font-family="Helvetica,sans-Serif" font-size="11.00">celery_task_id: str?</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="1942.5" y="-120.55" font-family="Helvetica,sans-Serif" font-size="11.00">priority: int</text>
|
<text xml:space="preserve" text-anchor="start" x="1859.25" y="-188.3" font-family="Helvetica,sans-Serif" font-size="11.00">execution_arn: str?</text>
|
||||||
<polyline fill="none" stroke="black" points="2056.25,-86.5 2056.25,-175.5"/>
|
<text xml:space="preserve" text-anchor="start" x="1859.25" y="-174.8" font-family="Helvetica,sans-Serif" font-size="11.00">priority: int</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="2064.25" y="-140.8" font-family="Helvetica,sans-Serif" font-size="11.00">created_at: datetime</text>
|
<polyline fill="none" stroke="black" points="1973,-147.5 1973,-236.5"/>
|
||||||
<text xml:space="preserve" text-anchor="start" x="2064.25" y="-127.3" font-family="Helvetica,sans-Serif" font-size="11.00">started_at: datetime?</text>
|
<text xml:space="preserve" text-anchor="start" x="1981" y="-201.8" font-family="Helvetica,sans-Serif" font-size="11.00">created_at: datetime</text>
|
||||||
<text xml:space="preserve" text-anchor="start" x="2064.25" y="-113.8" font-family="Helvetica,sans-Serif" font-size="11.00">completed_at: datetime?</text>
|
<text xml:space="preserve" text-anchor="start" x="1981" y="-188.3" font-family="Helvetica,sans-Serif" font-size="11.00">started_at: datetime?</text>
|
||||||
|
<text xml:space="preserve" text-anchor="start" x="1981" y="-174.8" font-family="Helvetica,sans-Serif" font-size="11.00">completed_at: datetime?</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- MediaAsset->TranscodeJob -->
|
<!-- MediaAsset->TranscodeJob -->
|
||||||
<g id="edge1" class="edge">
|
<g id="edge1" class="edge">
|
||||||
<title>MediaAsset->TranscodeJob</title>
|
<title>MediaAsset->TranscodeJob</title>
|
||||||
<path fill="none" stroke="black" d="M708.15,-147.67C708.15,-147.67 983.49,-147.67 983.49,-147.67"/>
|
<path fill="none" stroke="black" d="M708.33,-192C708.33,-192 900.24,-192 900.24,-192"/>
|
||||||
<polygon fill="black" stroke="black" points="983.49,-151.17 993.49,-147.67 983.49,-144.17 983.49,-151.17"/>
|
<polygon fill="black" stroke="black" points="900.24,-195.5 910.24,-192 900.24,-188.5 900.24,-195.5"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="910.62" y="-195.25" font-family="Helvetica,sans-Serif" font-size="10.00">1:N source_asset</text>
|
<text xml:space="preserve" text-anchor="middle" x="762.66" y="-182.5" font-family="Helvetica,sans-Serif" font-size="10.00">1:N source_asset</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- TranscodePreset -->
|
<!-- TranscodePreset -->
|
||||||
<g id="node2" class="node">
|
<g id="node2" class="node">
|
||||||
@@ -112,16 +113,16 @@
|
|||||||
<!-- TranscodePreset->TranscodeJob -->
|
<!-- TranscodePreset->TranscodeJob -->
|
||||||
<g id="edge2" class="edge">
|
<g id="edge2" class="edge">
|
||||||
<title>TranscodePreset->TranscodeJob</title>
|
<title>TranscodePreset->TranscodeJob</title>
|
||||||
<path fill="none" stroke="black" d="M766.5,-89.89C766.5,-101.97 766.5,-111.75 766.5,-111.75 766.5,-111.75 983.39,-111.75 983.39,-111.75"/>
|
<path fill="none" stroke="black" d="M767.25,-89.95C767.25,-125.61 767.25,-169.5 767.25,-169.5 767.25,-169.5 900.26,-169.5 900.26,-169.5"/>
|
||||||
<polygon fill="black" stroke="black" points="983.39,-115.25 993.39,-111.75 983.39,-108.25 983.39,-115.25"/>
|
<polygon fill="black" stroke="black" points="900.26,-173 910.26,-169.5 900.26,-166 900.26,-173"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="910.62" y="-48.25" font-family="Helvetica,sans-Serif" font-size="10.00">1:N preset</text>
|
<text xml:space="preserve" text-anchor="middle" x="768.85" y="-160" font-family="Helvetica,sans-Serif" font-size="10.00">1:N preset</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- TranscodeJob->MediaAsset -->
|
<!-- TranscodeJob->MediaAsset -->
|
||||||
<g id="edge3" class="edge">
|
<g id="edge3" class="edge">
|
||||||
<title>TranscodeJob->MediaAsset</title>
|
<title>TranscodeJob->MediaAsset</title>
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M995.06,-161.83C995.06,-161.83 719.99,-161.83 719.99,-161.83"/>
|
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M911.86,-214.5C911.86,-214.5 719.76,-214.5 719.76,-214.5"/>
|
||||||
<polygon fill="black" stroke="black" points="719.99,-158.33 709.99,-161.83 719.99,-165.33 719.99,-158.33"/>
|
<polygon fill="black" stroke="black" points="719.76,-211 709.76,-214.5 719.76,-218 719.76,-211"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="910.62" y="-134.25" font-family="Helvetica,sans-Serif" font-size="10.00">1:1 output_asset</text>
|
<text xml:space="preserve" text-anchor="middle" x="775.31" y="-205" font-family="Helvetica,sans-Serif" font-size="10.00">1:1 output_asset</text>
|
||||||
</g>
|
</g>
|
||||||
</g>
|
</g>
|
||||||
</svg>
|
</svg>
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 13 KiB |
@@ -3,7 +3,6 @@ digraph job_flow {
|
|||||||
node [shape=box, style=rounded, fontname="Helvetica"]
|
node [shape=box, style=rounded, fontname="Helvetica"]
|
||||||
edge [fontname="Helvetica", fontsize=10]
|
edge [fontname="Helvetica", fontsize=10]
|
||||||
|
|
||||||
// Title
|
|
||||||
labelloc="t"
|
labelloc="t"
|
||||||
label="MPR - Job Flow"
|
label="MPR - Job Flow"
|
||||||
fontsize=16
|
fontsize=16
|
||||||
@@ -11,7 +10,19 @@ digraph job_flow {
|
|||||||
|
|
||||||
graph [splines=ortho, nodesep=0.6, ranksep=0.6]
|
graph [splines=ortho, nodesep=0.6, ranksep=0.6]
|
||||||
|
|
||||||
// States
|
// API entry points
|
||||||
|
subgraph cluster_api {
|
||||||
|
label="API Entry Points"
|
||||||
|
style=dashed
|
||||||
|
color=gray
|
||||||
|
|
||||||
|
rest_create [label="POST /api/jobs/", shape=ellipse]
|
||||||
|
gql_create [label="mutation createJob", shape=ellipse]
|
||||||
|
rest_cancel [label="POST /api/jobs/{id}/cancel", shape=ellipse]
|
||||||
|
rest_callback [label="POST /api/jobs/{id}/callback", shape=ellipse]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job states
|
||||||
subgraph cluster_states {
|
subgraph cluster_states {
|
||||||
label="Job States"
|
label="Job States"
|
||||||
style=filled
|
style=filled
|
||||||
@@ -24,78 +35,70 @@ digraph job_flow {
|
|||||||
cancelled [label="CANCELLED", fillcolor="#6c757d", style="filled,rounded", fontcolor=white]
|
cancelled [label="CANCELLED", fillcolor="#6c757d", style="filled,rounded", fontcolor=white]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Transitions
|
// State transitions
|
||||||
pending -> processing [label="worker picks up"]
|
pending -> processing [xlabel="worker picks up"]
|
||||||
processing -> completed [label="success"]
|
processing -> completed [xlabel="success"]
|
||||||
processing -> failed [label="error"]
|
processing -> failed [xlabel="error"]
|
||||||
pending -> cancelled [label="user cancels"]
|
pending -> cancelled [xlabel="user cancels"]
|
||||||
processing -> cancelled [label="user cancels"]
|
processing -> cancelled [xlabel="user cancels"]
|
||||||
failed -> pending [label="retry"]
|
failed -> pending [xlabel="retry"]
|
||||||
|
|
||||||
// API actions
|
rest_create -> pending
|
||||||
subgraph cluster_api {
|
gql_create -> pending
|
||||||
label="API Actions"
|
rest_cancel -> cancelled [style=dashed]
|
||||||
style=dashed
|
|
||||||
color=gray
|
|
||||||
|
|
||||||
create_job [label="POST /jobs/", shape=ellipse]
|
// Executor dispatch
|
||||||
cancel_job [label="POST /jobs/{id}/cancel", shape=ellipse]
|
subgraph cluster_dispatch {
|
||||||
retry_job [label="POST /jobs/{id}/retry", shape=ellipse]
|
label="Executor Dispatch"
|
||||||
}
|
|
||||||
|
|
||||||
create_job -> pending
|
|
||||||
cancel_job -> cancelled [style=dashed]
|
|
||||||
retry_job -> pending [style=dashed]
|
|
||||||
|
|
||||||
// Executor layer
|
|
||||||
subgraph cluster_executor {
|
|
||||||
label="Executor Layer"
|
|
||||||
style=filled
|
style=filled
|
||||||
fillcolor="#fff8e8"
|
fillcolor="#fff8e8"
|
||||||
|
|
||||||
executor [label="Executor\n(abstract)", shape=diamond]
|
dispatch [label="MPR_EXECUTOR", shape=diamond]
|
||||||
local [label="LocalExecutor\nCelery + FFmpeg"]
|
|
||||||
lambda_exec [label="LambdaExecutor\nSQS + Lambda"]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
processing -> executor
|
pending -> dispatch
|
||||||
executor -> local [label="MPR_EXECUTOR=local"]
|
|
||||||
executor -> lambda_exec [label="MPR_EXECUTOR=lambda", style=dashed]
|
|
||||||
|
|
||||||
// FFmpeg operations
|
// Local path
|
||||||
subgraph cluster_ffmpeg {
|
subgraph cluster_local {
|
||||||
label="FFmpeg Operations"
|
label="Local Mode (Celery)"
|
||||||
style=filled
|
style=filled
|
||||||
fillcolor="#e8f4e8"
|
fillcolor="#e8f4e8"
|
||||||
|
|
||||||
transcode [label="Transcode\n(with preset)"]
|
celery_task [label="Celery Task\n(transcode queue)"]
|
||||||
trim [label="Trim\n(-c:v copy -c:a copy)"]
|
s3_download [label="S3 Download\n(MinIO)"]
|
||||||
|
ffmpeg_local [label="FFmpeg\ntranscode/trim"]
|
||||||
|
s3_upload [label="S3 Upload\n(MinIO)"]
|
||||||
|
db_update [label="DB Update\n(update_job_progress)"]
|
||||||
}
|
}
|
||||||
|
|
||||||
local -> transcode
|
dispatch -> celery_task [xlabel="local"]
|
||||||
local -> trim
|
celery_task -> s3_download
|
||||||
|
s3_download -> ffmpeg_local
|
||||||
|
ffmpeg_local -> s3_upload
|
||||||
|
s3_upload -> db_update
|
||||||
|
db_update -> completed [style=dotted]
|
||||||
|
|
||||||
// gRPC streaming
|
// Lambda path
|
||||||
subgraph cluster_grpc {
|
subgraph cluster_lambda {
|
||||||
label="gRPC Communication"
|
label="Lambda Mode (AWS)"
|
||||||
style=filled
|
style=filled
|
||||||
fillcolor="#e8e8f8"
|
fillcolor="#fde8d0"
|
||||||
|
|
||||||
grpc_stream [label="StreamProgress\n(server streaming)", shape=parallelogram]
|
sfn_start [label="Step Functions\nstart_execution"]
|
||||||
grpc_submit [label="SubmitJob\n(unary)", shape=parallelogram]
|
lambda_fn [label="Lambda\nFFmpeg container"]
|
||||||
grpc_cancel [label="CancelJob\n(unary)", shape=parallelogram]
|
s3_dl_aws [label="S3 Download\n(AWS)"]
|
||||||
|
ffmpeg_aws [label="FFmpeg\ntranscode/trim"]
|
||||||
|
s3_ul_aws [label="S3 Upload\n(AWS)"]
|
||||||
|
callback [label="HTTP Callback\nPOST /jobs/{id}/callback"]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Progress tracking via gRPC
|
dispatch -> sfn_start [xlabel="lambda"]
|
||||||
progress [label="Progress Updates\n(gRPC → Redis → DB)", shape=note]
|
sfn_start -> lambda_fn
|
||||||
transcode -> progress [style=dotted]
|
lambda_fn -> s3_dl_aws
|
||||||
trim -> progress [style=dotted]
|
s3_dl_aws -> ffmpeg_aws
|
||||||
progress -> grpc_stream [style=dotted, label="stream to client"]
|
ffmpeg_aws -> s3_ul_aws
|
||||||
grpc_stream -> processing [style=dotted, label="update status"]
|
s3_ul_aws -> callback
|
||||||
|
callback -> completed [style=dotted]
|
||||||
|
|
||||||
// gRPC job control
|
rest_callback -> completed [style=dashed, xlabel="Lambda reports"]
|
||||||
create_job -> grpc_submit [label="via gRPC"]
|
|
||||||
grpc_submit -> pending [style=dashed]
|
|
||||||
cancel_job -> grpc_cancel [label="via gRPC"]
|
|
||||||
grpc_cancel -> cancelled [style=dashed]
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,296 +1,329 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||||
<!-- Generated by graphviz version 14.1.1 (0)
|
<!-- Generated by graphviz version 14.1.2 (0)
|
||||||
-->
|
-->
|
||||||
<!-- Title: job_flow Pages: 1 -->
|
<!-- Title: job_flow Pages: 1 -->
|
||||||
<svg width="1398pt" height="843pt"
|
<svg width="1621pt" height="655pt"
|
||||||
viewBox="0.00 0.00 1398.00 843.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
viewBox="0.00 0.00 1621.00 655.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 838.75)">
|
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 650.5)">
|
||||||
<title>job_flow</title>
|
<title>job_flow</title>
|
||||||
<polygon fill="white" stroke="none" points="-4,4 -4,-838.75 1394,-838.75 1394,4 -4,4"/>
|
<polygon fill="white" stroke="none" points="-4,4 -4,-650.5 1617,-650.5 1617,4 -4,4"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="695" y="-815.55" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - Job Flow</text>
|
<text xml:space="preserve" text-anchor="middle" x="806.5" y="-627.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">MPR - Job Flow</text>
|
||||||
<g id="clust1" class="cluster">
|
<g id="clust1" class="cluster">
|
||||||
<title>cluster_states</title>
|
<title>cluster_api</title>
|
||||||
<polygon fill="#f8f8f8" stroke="black" points="774,-8 774,-297.5 1154,-297.5 1154,-8 774,-8"/>
|
<polygon fill="none" stroke="gray" stroke-dasharray="5,2" points="297,-269.75 297,-349.25 1395,-349.25 1395,-269.75 297,-269.75"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="964" y="-278.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Job States</text>
|
<text xml:space="preserve" text-anchor="middle" x="846" y="-330.05" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">API Entry Points</text>
|
||||||
</g>
|
</g>
|
||||||
<g id="clust2" class="cluster">
|
<g id="clust2" class="cluster">
|
||||||
<title>cluster_api</title>
|
<title>cluster_states</title>
|
||||||
<polygon fill="none" stroke="gray" stroke-dasharray="5,2" points="674,-360 674,-439.5 1382,-439.5 1382,-360 674,-360"/>
|
<polygon fill="#f8f8f8" stroke="black" points="572,-11.25 572,-261.75 939,-261.75 939,-11.25 572,-11.25"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1028" y="-420.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">API Actions</text>
|
<text xml:space="preserve" text-anchor="middle" x="755.5" y="-242.55" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Job States</text>
|
||||||
</g>
|
</g>
|
||||||
<g id="clust3" class="cluster">
|
<g id="clust3" class="cluster">
|
||||||
<title>cluster_executor</title>
|
<title>cluster_dispatch</title>
|
||||||
<polygon fill="#fff8e8" stroke="black" points="8,-571.5 8,-799.25 352,-799.25 352,-571.5 8,-571.5"/>
|
<polygon fill="#fff8e8" stroke="black" points="103,-531.5 103,-611 377,-611 377,-531.5 103,-531.5"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="180" y="-780.05" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Executor Layer</text>
|
<text xml:space="preserve" text-anchor="middle" x="240" y="-591.8" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Executor Dispatch</text>
|
||||||
</g>
|
</g>
|
||||||
<g id="clust4" class="cluster">
|
<g id="clust4" class="cluster">
|
||||||
<title>cluster_ffmpeg</title>
|
<title>cluster_local</title>
|
||||||
<polygon fill="#e8f4e8" stroke="black" points="73,-462.5 73,-548.5 393,-548.5 393,-462.5 73,-462.5"/>
|
<polygon fill="#e8f4e8" stroke="black" points="8,-93.5 8,-523.5 203,-523.5 203,-93.5 8,-93.5"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="233" y="-529.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">FFmpeg Operations</text>
|
<text xml:space="preserve" text-anchor="middle" x="105.5" y="-504.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Local Mode (Celery)</text>
|
||||||
</g>
|
</g>
|
||||||
<g id="clust5" class="cluster">
|
<g id="clust5" class="cluster">
|
||||||
<title>cluster_grpc</title>
|
<title>cluster_lambda</title>
|
||||||
<polygon fill="#e8e8f8" stroke="black" points="8,-193.5 8,-322 766,-322 766,-193.5 8,-193.5"/>
|
<polygon fill="#fde8d0" stroke="black" points="1403,-8 1403,-523.5 1605,-523.5 1605,-8 1403,-8"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="387" y="-302.8" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">gRPC Communication</text>
|
<text xml:space="preserve" text-anchor="middle" x="1504" y="-504.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="16.00">Lambda Mode (AWS)</text>
|
||||||
|
</g>
|
||||||
|
<!-- rest_create -->
|
||||||
|
<g id="node1" class="node">
|
||||||
|
<title>rest_create</title>
|
||||||
|
<ellipse fill="none" stroke="black" cx="389" cy="-295.75" rx="84.35" ry="18"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="389" y="-291.07" font-family="Helvetica,sans-Serif" font-size="14.00">POST /api/jobs/</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- pending -->
|
<!-- pending -->
|
||||||
<g id="node1" class="node">
|
<g id="node5" class="node">
|
||||||
<title>pending</title>
|
<title>pending</title>
|
||||||
<path fill="#ffc107" stroke="black" d="M971.88,-262C971.88,-262 916.12,-262 916.12,-262 910.12,-262 904.12,-256 904.12,-250 904.12,-250 904.12,-238 904.12,-238 904.12,-232 910.12,-226 916.12,-226 916.12,-226 971.88,-226 971.88,-226 977.88,-226 983.88,-232 983.88,-238 983.88,-238 983.88,-250 983.88,-250 983.88,-256 977.88,-262 971.88,-262"/>
|
<path fill="#ffc107" stroke="black" d="M647.88,-226.25C647.88,-226.25 592.12,-226.25 592.12,-226.25 586.12,-226.25 580.12,-220.25 580.12,-214.25 580.12,-214.25 580.12,-202.25 580.12,-202.25 580.12,-196.25 586.12,-190.25 592.12,-190.25 592.12,-190.25 647.88,-190.25 647.88,-190.25 653.88,-190.25 659.88,-196.25 659.88,-202.25 659.88,-202.25 659.88,-214.25 659.88,-214.25 659.88,-220.25 653.88,-226.25 647.88,-226.25"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="944" y="-239.32" font-family="Helvetica,sans-Serif" font-size="14.00">PENDING</text>
|
<text xml:space="preserve" text-anchor="middle" x="620" y="-203.57" font-family="Helvetica,sans-Serif" font-size="14.00">PENDING</text>
|
||||||
|
</g>
|
||||||
|
<!-- rest_create->pending -->
|
||||||
|
<g id="edge7" class="edge">
|
||||||
|
<title>rest_create->pending</title>
|
||||||
|
<path fill="none" stroke="black" d="M389,-277.61C389,-253.52 389,-214 389,-214 389,-214 568.25,-214 568.25,-214"/>
|
||||||
|
<polygon fill="black" stroke="black" points="568.25,-217.5 578.25,-214 568.25,-210.5 568.25,-217.5"/>
|
||||||
|
</g>
|
||||||
|
<!-- gql_create -->
|
||||||
|
<g id="node2" class="node">
|
||||||
|
<title>gql_create</title>
|
||||||
|
<ellipse fill="none" stroke="black" cx="620" cy="-295.75" rx="103.29" ry="18"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="620" y="-291.07" font-family="Helvetica,sans-Serif" font-size="14.00">mutation createJob</text>
|
||||||
|
</g>
|
||||||
|
<!-- gql_create->pending -->
|
||||||
|
<g id="edge8" class="edge">
|
||||||
|
<title>gql_create->pending</title>
|
||||||
|
<path fill="none" stroke="black" d="M620,-277.62C620,-277.62 620,-238.17 620,-238.17"/>
|
||||||
|
<polygon fill="black" stroke="black" points="623.5,-238.17 620,-228.17 616.5,-238.17 623.5,-238.17"/>
|
||||||
|
</g>
|
||||||
|
<!-- rest_cancel -->
|
||||||
|
<g id="node3" class="node">
|
||||||
|
<title>rest_cancel</title>
|
||||||
|
<ellipse fill="none" stroke="black" cx="1247" cy="-295.75" rx="140.12" ry="18"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1247" y="-291.07" font-family="Helvetica,sans-Serif" font-size="14.00">POST /api/jobs/{id}/cancel</text>
|
||||||
|
</g>
|
||||||
|
<!-- cancelled -->
|
||||||
|
<g id="node9" class="node">
|
||||||
|
<title>cancelled</title>
|
||||||
|
<path fill="#6c757d" stroke="black" d="M918.62,-55.25C918.62,-55.25 843.38,-55.25 843.38,-55.25 837.38,-55.25 831.38,-49.25 831.38,-43.25 831.38,-43.25 831.38,-31.25 831.38,-31.25 831.38,-25.25 837.38,-19.25 843.38,-19.25 843.38,-19.25 918.62,-19.25 918.62,-19.25 924.62,-19.25 930.62,-25.25 930.62,-31.25 930.62,-31.25 930.62,-43.25 930.62,-43.25 930.62,-49.25 924.62,-55.25 918.62,-55.25"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="881" y="-32.58" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">CANCELLED</text>
|
||||||
|
</g>
|
||||||
|
<!-- rest_cancel->cancelled -->
|
||||||
|
<g id="edge9" class="edge">
|
||||||
|
<title>rest_cancel->cancelled</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M1247,-277.56C1247,-218.66 1247,-37 1247,-37 1247,-37 942.64,-37 942.64,-37"/>
|
||||||
|
<polygon fill="black" stroke="black" points="942.64,-33.5 932.64,-37 942.64,-40.5 942.64,-33.5"/>
|
||||||
|
</g>
|
||||||
|
<!-- rest_callback -->
|
||||||
|
<g id="node4" class="node">
|
||||||
|
<title>rest_callback</title>
|
||||||
|
<ellipse fill="none" stroke="black" cx="915" cy="-295.75" rx="148.54" ry="18"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="915" y="-291.07" font-family="Helvetica,sans-Serif" font-size="14.00">POST /api/jobs/{id}/callback</text>
|
||||||
|
</g>
|
||||||
|
<!-- completed -->
|
||||||
|
<g id="node7" class="node">
|
||||||
|
<title>completed</title>
|
||||||
|
<path fill="#28a745" stroke="black" d="M776.75,-55.25C776.75,-55.25 699.25,-55.25 699.25,-55.25 693.25,-55.25 687.25,-49.25 687.25,-43.25 687.25,-43.25 687.25,-31.25 687.25,-31.25 687.25,-25.25 693.25,-19.25 699.25,-19.25 699.25,-19.25 776.75,-19.25 776.75,-19.25 782.75,-19.25 788.75,-25.25 788.75,-31.25 788.75,-31.25 788.75,-43.25 788.75,-43.25 788.75,-49.25 782.75,-55.25 776.75,-55.25"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="738" y="-32.58" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">COMPLETED</text>
|
||||||
|
</g>
|
||||||
|
<!-- rest_callback->completed -->
|
||||||
|
<g id="edge24" class="edge">
|
||||||
|
<title>rest_callback->completed</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M783.42,-287.15C783.42,-287.15 783.42,-67.24 783.42,-67.24"/>
|
||||||
|
<polygon fill="black" stroke="black" points="786.92,-67.24 783.42,-57.24 779.92,-67.24 786.92,-67.24"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="745.17" y="-180.44" font-family="Helvetica,sans-Serif" font-size="10.00">Lambda reports</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- processing -->
|
<!-- processing -->
|
||||||
<g id="node2" class="node">
|
<g id="node6" class="node">
|
||||||
<title>processing</title>
|
<title>processing</title>
|
||||||
<path fill="#17a2b8" stroke="black" d="M877.75,-144.75C877.75,-144.75 794.25,-144.75 794.25,-144.75 788.25,-144.75 782.25,-138.75 782.25,-132.75 782.25,-132.75 782.25,-120.75 782.25,-120.75 782.25,-114.75 788.25,-108.75 794.25,-108.75 794.25,-108.75 877.75,-108.75 877.75,-108.75 883.75,-108.75 889.75,-114.75 889.75,-120.75 889.75,-120.75 889.75,-132.75 889.75,-132.75 889.75,-138.75 883.75,-144.75 877.75,-144.75"/>
|
<path fill="#17a2b8" stroke="black" d="M768.75,-140.75C768.75,-140.75 685.25,-140.75 685.25,-140.75 679.25,-140.75 673.25,-134.75 673.25,-128.75 673.25,-128.75 673.25,-116.75 673.25,-116.75 673.25,-110.75 679.25,-104.75 685.25,-104.75 685.25,-104.75 768.75,-104.75 768.75,-104.75 774.75,-104.75 780.75,-110.75 780.75,-116.75 780.75,-116.75 780.75,-128.75 780.75,-128.75 780.75,-134.75 774.75,-140.75 768.75,-140.75"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="836" y="-122.08" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">PROCESSING</text>
|
<text xml:space="preserve" text-anchor="middle" x="727" y="-118.08" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">PROCESSING</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- pending->processing -->
|
<!-- pending->processing -->
|
||||||
<g id="edge1" class="edge">
|
<g id="edge1" class="edge">
|
||||||
<title>pending->processing</title>
|
<title>pending->processing</title>
|
||||||
<path fill="none" stroke="black" d="M920.04,-225.68C920.04,-194.87 920.04,-136 920.04,-136 920.04,-136 901.69,-136 901.69,-136"/>
|
<path fill="none" stroke="black" d="M654.58,-189.87C654.58,-166.46 654.58,-129 654.58,-129 654.58,-129 661.34,-129 661.34,-129"/>
|
||||||
<polygon fill="black" stroke="black" points="901.69,-132.5 891.69,-136 901.69,-139.5 901.69,-132.5"/>
|
<polygon fill="black" stroke="black" points="661.34,-132.5 671.34,-129 661.34,-125.5 661.34,-132.5"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="902.25" y="-170" font-family="Helvetica,sans-Serif" font-size="10.00">worker picks up</text>
|
<text xml:space="preserve" text-anchor="middle" x="616.33" y="-159.3" font-family="Helvetica,sans-Serif" font-size="10.00">worker picks up</text>
|
||||||
</g>
|
|
||||||
<!-- cancelled -->
|
|
||||||
<g id="node5" class="node">
|
|
||||||
<title>cancelled</title>
|
|
||||||
<path fill="#6c757d" stroke="black" d="M1122.62,-52C1122.62,-52 1047.38,-52 1047.38,-52 1041.38,-52 1035.38,-46 1035.38,-40 1035.38,-40 1035.38,-28 1035.38,-28 1035.38,-22 1041.38,-16 1047.38,-16 1047.38,-16 1122.62,-16 1122.62,-16 1128.62,-16 1134.62,-22 1134.62,-28 1134.62,-28 1134.62,-40 1134.62,-40 1134.62,-46 1128.62,-52 1122.62,-52"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1085" y="-29.32" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">CANCELLED</text>
|
|
||||||
</g>
|
</g>
|
||||||
<!-- pending->cancelled -->
|
<!-- pending->cancelled -->
|
||||||
<g id="edge4" class="edge">
|
<g id="edge4" class="edge">
|
||||||
<title>pending->cancelled</title>
|
<title>pending->cancelled</title>
|
||||||
<path fill="none" stroke="black" d="M984.17,-238C1022.83,-238 1075.49,-238 1075.49,-238 1075.49,-238 1075.49,-63.98 1075.49,-63.98"/>
|
<path fill="none" stroke="black" d="M660.36,-208C737.33,-208 897.54,-208 897.54,-208 897.54,-208 897.54,-67.04 897.54,-67.04"/>
|
||||||
<polygon fill="black" stroke="black" points="1078.99,-63.98 1075.49,-53.98 1071.99,-63.98 1078.99,-63.98"/>
|
<polygon fill="black" stroke="black" points="901.04,-67.04 897.54,-57.04 894.04,-67.04 901.04,-67.04"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1115.38" y="-123.62" font-family="Helvetica,sans-Serif" font-size="10.00">user cancels</text>
|
<text xml:space="preserve" text-anchor="middle" x="819.06" y="-211.25" font-family="Helvetica,sans-Serif" font-size="10.00">user cancels</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- completed -->
|
<!-- dispatch -->
|
||||||
<g id="node3" class="node">
|
<g id="node10" class="node">
|
||||||
<title>completed</title>
|
<title>dispatch</title>
|
||||||
<path fill="#28a745" stroke="black" d="M871.75,-52C871.75,-52 794.25,-52 794.25,-52 788.25,-52 782.25,-46 782.25,-40 782.25,-40 782.25,-28 782.25,-28 782.25,-22 788.25,-16 794.25,-16 794.25,-16 871.75,-16 871.75,-16 877.75,-16 883.75,-22 883.75,-28 883.75,-28 883.75,-40 883.75,-40 883.75,-46 877.75,-52 871.75,-52"/>
|
<path fill="none" stroke="black" d="M228.12,-573.84C228.12,-573.84 122.92,-559.16 122.92,-559.16 116.98,-558.33 116.98,-556.67 122.92,-555.84 122.92,-555.84 228.12,-541.16 228.12,-541.16 234.06,-540.33 245.94,-540.33 251.88,-541.16 251.88,-541.16 357.08,-555.84 357.08,-555.84 363.02,-556.67 363.02,-558.33 357.08,-559.16 357.08,-559.16 251.88,-573.84 251.88,-573.84 245.94,-574.67 234.06,-574.67 228.12,-573.84"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="833" y="-29.32" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">COMPLETED</text>
|
<text xml:space="preserve" text-anchor="middle" x="240" y="-552.83" font-family="Helvetica,sans-Serif" font-size="14.00">MPR_EXECUTOR</text>
|
||||||
|
</g>
|
||||||
|
<!-- pending->dispatch -->
|
||||||
|
<g id="edge10" class="edge">
|
||||||
|
<title>pending->dispatch</title>
|
||||||
|
<path fill="none" stroke="black" d="M579.92,-202C483.92,-202 248.76,-202 248.76,-202 248.76,-202 248.76,-528.84 248.76,-528.84"/>
|
||||||
|
<polygon fill="black" stroke="black" points="245.26,-528.84 248.76,-538.84 252.26,-528.84 245.26,-528.84"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- processing->completed -->
|
<!-- processing->completed -->
|
||||||
<g id="edge2" class="edge">
|
<g id="edge2" class="edge">
|
||||||
<title>processing->completed</title>
|
<title>processing->completed</title>
|
||||||
<path fill="none" stroke="black" d="M833,-108.43C833,-108.43 833,-63.8 833,-63.8"/>
|
<path fill="none" stroke="black" d="M734,-104.62C734,-104.62 734,-67.16 734,-67.16"/>
|
||||||
<polygon fill="black" stroke="black" points="836.5,-63.8 833,-53.8 829.5,-63.8 836.5,-63.8"/>
|
<polygon fill="black" stroke="black" points="737.5,-67.16 734,-57.16 730.5,-67.16 737.5,-67.16"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="844.12" y="-77.25" font-family="Helvetica,sans-Serif" font-size="10.00">success</text>
|
<text xml:space="preserve" text-anchor="middle" x="714.88" y="-89.14" font-family="Helvetica,sans-Serif" font-size="10.00">success</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- failed -->
|
<!-- failed -->
|
||||||
<g id="node4" class="node">
|
<g id="node8" class="node">
|
||||||
<title>failed</title>
|
<title>failed</title>
|
||||||
<path fill="#dc3545" stroke="black" d="M980,-52C980,-52 940,-52 940,-52 934,-52 928,-46 928,-40 928,-40 928,-28 928,-28 928,-22 934,-16 940,-16 940,-16 980,-16 980,-16 986,-16 992,-22 992,-28 992,-28 992,-40 992,-40 992,-46 986,-52 980,-52"/>
|
<path fill="#dc3545" stroke="black" d="M632,-55.25C632,-55.25 592,-55.25 592,-55.25 586,-55.25 580,-49.25 580,-43.25 580,-43.25 580,-31.25 580,-31.25 580,-25.25 586,-19.25 592,-19.25 592,-19.25 632,-19.25 632,-19.25 638,-19.25 644,-25.25 644,-31.25 644,-31.25 644,-43.25 644,-43.25 644,-49.25 638,-55.25 632,-55.25"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="960" y="-29.32" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">FAILED</text>
|
<text xml:space="preserve" text-anchor="middle" x="612" y="-32.58" font-family="Helvetica,sans-Serif" font-size="14.00" fill="white">FAILED</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- processing->failed -->
|
<!-- processing->failed -->
|
||||||
<g id="edge3" class="edge">
|
<g id="edge3" class="edge">
|
||||||
<title>processing->failed</title>
|
<title>processing->failed</title>
|
||||||
<path fill="none" stroke="black" d="M890.02,-118C918.1,-118 946.62,-118 946.62,-118 946.62,-118 946.62,-63.74 946.62,-63.74"/>
|
<path fill="none" stroke="black" d="M680.25,-104.62C680.25,-77.88 680.25,-31 680.25,-31 680.25,-31 655.64,-31 655.64,-31"/>
|
||||||
<polygon fill="black" stroke="black" points="950.13,-63.74 946.63,-53.74 943.13,-63.74 950.13,-63.74"/>
|
<polygon fill="black" stroke="black" points="655.64,-27.5 645.64,-31 655.64,-34.5 655.64,-27.5"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="922.62" y="-77.25" font-family="Helvetica,sans-Serif" font-size="10.00">error</text>
|
<text xml:space="preserve" text-anchor="middle" x="668.62" y="-58.76" font-family="Helvetica,sans-Serif" font-size="10.00">error</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- processing->cancelled -->
|
<!-- processing->cancelled -->
|
||||||
<g id="edge5" class="edge">
|
<g id="edge5" class="edge">
|
||||||
<title>processing->cancelled</title>
|
<title>processing->cancelled</title>
|
||||||
<path fill="none" stroke="black" d="M890.24,-127C953.27,-127 1048.75,-127 1048.75,-127 1048.75,-127 1048.75,-63.89 1048.75,-63.89"/>
|
<path fill="none" stroke="black" d="M780.93,-123C819.44,-123 864.46,-123 864.46,-123 864.46,-123 864.46,-66.95 864.46,-66.95"/>
|
||||||
<polygon fill="black" stroke="black" points="1052.25,-63.89 1048.75,-53.89 1045.25,-63.89 1052.25,-63.89"/>
|
<polygon fill="black" stroke="black" points="867.96,-66.95 864.46,-56.95 860.96,-66.95 867.96,-66.95"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1012.38" y="-77.25" font-family="Helvetica,sans-Serif" font-size="10.00">user cancels</text>
|
<text xml:space="preserve" text-anchor="middle" x="820.35" y="-126.25" font-family="Helvetica,sans-Serif" font-size="10.00">user cancels</text>
|
||||||
</g>
|
|
||||||
<!-- executor -->
|
|
||||||
<g id="node9" class="node">
|
|
||||||
<title>executor</title>
|
|
||||||
<path fill="none" stroke="black" d="M89.31,-758.31C89.31,-758.31 27.19,-726.69 27.19,-726.69 21.85,-723.97 21.85,-718.53 27.19,-715.81 27.19,-715.81 89.31,-684.19 89.31,-684.19 94.65,-681.47 105.35,-681.47 110.69,-684.19 110.69,-684.19 172.81,-715.81 172.81,-715.81 178.15,-718.53 178.15,-723.97 172.81,-726.69 172.81,-726.69 110.69,-758.31 110.69,-758.31 105.35,-761.03 94.65,-761.03 89.31,-758.31"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="100" y="-725.2" font-family="Helvetica,sans-Serif" font-size="14.00">Executor</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="100" y="-707.95" font-family="Helvetica,sans-Serif" font-size="14.00">(abstract)</text>
|
|
||||||
</g>
|
|
||||||
<!-- processing->executor -->
|
|
||||||
<g id="edge10" class="edge">
|
|
||||||
<title>processing->executor</title>
|
|
||||||
<path fill="none" stroke="black" d="M836.12,-145.19C836.12,-245.49 836.12,-721 836.12,-721 836.12,-721 195.6,-721 195.6,-721"/>
|
|
||||||
<polygon fill="black" stroke="black" points="195.6,-717.5 185.6,-721 195.6,-724.5 195.6,-717.5"/>
|
|
||||||
</g>
|
</g>
|
||||||
<!-- failed->pending -->
|
<!-- failed->pending -->
|
||||||
<g id="edge6" class="edge">
|
<g id="edge6" class="edge">
|
||||||
<title>failed->pending</title>
|
<title>failed->pending</title>
|
||||||
<path fill="none" stroke="black" d="M965.25,-52.27C965.25,-52.27 965.25,-214.11 965.25,-214.11"/>
|
<path fill="none" stroke="black" d="M612.06,-55.55C612.06,-55.55 612.06,-178.31 612.06,-178.31"/>
|
||||||
<polygon fill="black" stroke="black" points="961.75,-214.11 965.25,-224.11 968.75,-214.11 961.75,-214.11"/>
|
<polygon fill="black" stroke="black" points="608.56,-178.31 612.06,-188.31 615.56,-178.31 608.56,-178.31"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="987.62" y="-123.62" font-family="Helvetica,sans-Serif" font-size="10.00">retry</text>
|
<text xml:space="preserve" text-anchor="middle" x="600.44" y="-120.18" font-family="Helvetica,sans-Serif" font-size="10.00">retry</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- create_job -->
|
<!-- celery_task -->
|
||||||
<g id="node6" class="node">
|
|
||||||
<title>create_job</title>
|
|
||||||
<ellipse fill="none" stroke="black" cx="748" cy="-386" rx="66.47" ry="18"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="748" y="-381.32" font-family="Helvetica,sans-Serif" font-size="14.00">POST /jobs/</text>
|
|
||||||
</g>
|
|
||||||
<!-- create_job->pending -->
|
|
||||||
<g id="edge7" class="edge">
|
|
||||||
<title>create_job->pending</title>
|
|
||||||
<path fill="none" stroke="black" d="M798.36,-373.89C798.36,-339.55 798.36,-244 798.36,-244 798.36,-244 892.3,-244 892.3,-244"/>
|
|
||||||
<polygon fill="black" stroke="black" points="892.3,-247.5 902.3,-244 892.3,-240.5 892.3,-247.5"/>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_submit -->
|
|
||||||
<g id="node15" class="node">
|
|
||||||
<title>grpc_submit</title>
|
|
||||||
<path fill="none" stroke="black" d="M528.46,-286.5C528.46,-286.5 408.56,-286.5 408.56,-286.5 402.56,-286.5 394.16,-281 391.77,-275.5 391.77,-275.5 364.33,-212.5 364.33,-212.5 361.94,-207 365.54,-201.5 371.54,-201.5 371.54,-201.5 491.44,-201.5 491.44,-201.5 497.44,-201.5 505.84,-207 508.23,-212.5 508.23,-212.5 535.67,-275.5 535.67,-275.5 538.06,-281 534.46,-286.5 528.46,-286.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="450" y="-247.95" font-family="Helvetica,sans-Serif" font-size="14.00">SubmitJob</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="450" y="-230.7" font-family="Helvetica,sans-Serif" font-size="14.00">(unary)</text>
|
|
||||||
</g>
|
|
||||||
<!-- create_job->grpc_submit -->
|
|
||||||
<g id="edge19" class="edge">
|
|
||||||
<title>create_job->grpc_submit</title>
|
|
||||||
<path fill="none" stroke="black" d="M681.06,-386C596.67,-386 462.48,-386 462.48,-386 462.48,-386 462.48,-298.5 462.48,-298.5"/>
|
|
||||||
<polygon fill="black" stroke="black" points="465.98,-298.5 462.48,-288.5 458.98,-298.5 465.98,-298.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="620.75" y="-333.25" font-family="Helvetica,sans-Serif" font-size="10.00">via gRPC</text>
|
|
||||||
</g>
|
|
||||||
<!-- cancel_job -->
|
|
||||||
<g id="node7" class="node">
|
|
||||||
<title>cancel_job</title>
|
|
||||||
<ellipse fill="none" stroke="black" cx="980" cy="-386" rx="122.23" ry="18"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="980" y="-381.32" font-family="Helvetica,sans-Serif" font-size="14.00">POST /jobs/{id}/cancel</text>
|
|
||||||
</g>
|
|
||||||
<!-- cancel_job->cancelled -->
|
|
||||||
<g id="edge8" class="edge">
|
|
||||||
<title>cancel_job->cancelled</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M1088.86,-377.65C1088.86,-377.65 1088.86,-63.86 1088.86,-63.86"/>
|
|
||||||
<polygon fill="black" stroke="black" points="1092.36,-63.86 1088.86,-53.86 1085.36,-63.86 1092.36,-63.86"/>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_cancel -->
|
|
||||||
<g id="node16" class="node">
|
|
||||||
<title>grpc_cancel</title>
|
|
||||||
<path fill="none" stroke="black" d="M746.35,-286.5C746.35,-286.5 631.4,-286.5 631.4,-286.5 625.4,-286.5 617.07,-280.97 614.75,-275.44 614.75,-275.44 588.31,-212.56 588.31,-212.56 585.98,-207.03 589.65,-201.5 595.65,-201.5 595.65,-201.5 710.6,-201.5 710.6,-201.5 716.6,-201.5 724.93,-207.03 727.25,-212.56 727.25,-212.56 753.69,-275.44 753.69,-275.44 756.02,-280.97 752.35,-286.5 746.35,-286.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="671" y="-247.95" font-family="Helvetica,sans-Serif" font-size="14.00">CancelJob</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="671" y="-230.7" font-family="Helvetica,sans-Serif" font-size="14.00">(unary)</text>
|
|
||||||
</g>
|
|
||||||
<!-- cancel_job->grpc_cancel -->
|
|
||||||
<g id="edge21" class="edge">
|
|
||||||
<title>cancel_job->grpc_cancel</title>
|
|
||||||
<path fill="none" stroke="black" d="M873.76,-376.83C873.76,-350.09 873.76,-274 873.76,-274 873.76,-274 764.98,-274 764.98,-274"/>
|
|
||||||
<polygon fill="black" stroke="black" points="764.98,-270.5 754.98,-274 764.98,-277.5 764.98,-270.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="870.75" y="-333.25" font-family="Helvetica,sans-Serif" font-size="10.00">via gRPC</text>
|
|
||||||
</g>
|
|
||||||
<!-- retry_job -->
|
|
||||||
<g id="node8" class="node">
|
|
||||||
<title>retry_job</title>
|
|
||||||
<ellipse fill="none" stroke="black" cx="1260" cy="-386" rx="114.34" ry="18"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1260" y="-381.32" font-family="Helvetica,sans-Serif" font-size="14.00">POST /jobs/{id}/retry</text>
|
|
||||||
</g>
|
|
||||||
<!-- retry_job->pending -->
|
|
||||||
<g id="edge9" class="edge">
|
|
||||||
<title>retry_job->pending</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M1260,-367.66C1260,-330.54 1260,-250 1260,-250 1260,-250 995.86,-250 995.86,-250"/>
|
|
||||||
<polygon fill="black" stroke="black" points="995.86,-246.5 985.86,-250 995.86,-253.5 995.86,-246.5"/>
|
|
||||||
</g>
|
|
||||||
<!-- local -->
|
|
||||||
<g id="node10" class="node">
|
|
||||||
<title>local</title>
|
|
||||||
<path fill="none" stroke="black" d="M316.75,-622C316.75,-622 203.25,-622 203.25,-622 197.25,-622 191.25,-616 191.25,-610 191.25,-610 191.25,-591.5 191.25,-591.5 191.25,-585.5 197.25,-579.5 203.25,-579.5 203.25,-579.5 316.75,-579.5 316.75,-579.5 322.75,-579.5 328.75,-585.5 328.75,-591.5 328.75,-591.5 328.75,-610 328.75,-610 328.75,-616 322.75,-622 316.75,-622"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="260" y="-604.7" font-family="Helvetica,sans-Serif" font-size="14.00">LocalExecutor</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="260" y="-587.45" font-family="Helvetica,sans-Serif" font-size="14.00">Celery + FFmpeg</text>
|
|
||||||
</g>
|
|
||||||
<!-- executor->local -->
|
|
||||||
<g id="edge11" class="edge">
|
|
||||||
<title>executor->local</title>
|
|
||||||
<path fill="none" stroke="black" d="M165.81,-711.81C165.81,-683.47 165.81,-601 165.81,-601 165.81,-601 179.54,-601 179.54,-601"/>
|
|
||||||
<polygon fill="black" stroke="black" points="179.54,-604.5 189.54,-601 179.54,-597.5 179.54,-604.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="287.88" y="-647.25" font-family="Helvetica,sans-Serif" font-size="10.00">MPR_EXECUTOR=local</text>
|
|
||||||
</g>
|
|
||||||
<!-- lambda_exec -->
|
|
||||||
<g id="node11" class="node">
|
<g id="node11" class="node">
|
||||||
<title>lambda_exec</title>
|
<title>celery_task</title>
|
||||||
<path fill="none" stroke="black" d="M136.12,-622C136.12,-622 27.88,-622 27.88,-622 21.88,-622 15.88,-616 15.88,-610 15.88,-610 15.88,-591.5 15.88,-591.5 15.88,-585.5 21.88,-579.5 27.88,-579.5 27.88,-579.5 136.12,-579.5 136.12,-579.5 142.12,-579.5 148.12,-585.5 148.12,-591.5 148.12,-591.5 148.12,-610 148.12,-610 148.12,-616 142.12,-622 136.12,-622"/>
|
<path fill="none" stroke="black" d="M162.75,-488C162.75,-488 43.25,-488 43.25,-488 37.25,-488 31.25,-482 31.25,-476 31.25,-476 31.25,-457.5 31.25,-457.5 31.25,-451.5 37.25,-445.5 43.25,-445.5 43.25,-445.5 162.75,-445.5 162.75,-445.5 168.75,-445.5 174.75,-451.5 174.75,-457.5 174.75,-457.5 174.75,-476 174.75,-476 174.75,-482 168.75,-488 162.75,-488"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="82" y="-604.7" font-family="Helvetica,sans-Serif" font-size="14.00">LambdaExecutor</text>
|
<text xml:space="preserve" text-anchor="middle" x="103" y="-470.7" font-family="Helvetica,sans-Serif" font-size="14.00">Celery Task</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="82" y="-587.45" font-family="Helvetica,sans-Serif" font-size="14.00">SQS + Lambda</text>
|
<text xml:space="preserve" text-anchor="middle" x="103" y="-453.45" font-family="Helvetica,sans-Serif" font-size="14.00">(transcode queue)</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- executor->lambda_exec -->
|
<!-- dispatch->celery_task -->
|
||||||
<g id="edge12" class="edge">
|
<g id="edge11" class="edge">
|
||||||
<title>executor->lambda_exec</title>
|
<title>dispatch->celery_task</title>
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M82.31,-687.36C82.31,-687.36 82.31,-633.77 82.31,-633.77"/>
|
<path fill="none" stroke="black" d="M142.89,-552.62C142.89,-552.62 142.89,-499.67 142.89,-499.67"/>
|
||||||
<polygon fill="black" stroke="black" points="85.81,-633.77 82.31,-623.77 78.81,-633.77 85.81,-633.77"/>
|
<polygon fill="black" stroke="black" points="146.39,-499.67 142.89,-489.67 139.39,-499.67 146.39,-499.67"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="121.62" y="-647.25" font-family="Helvetica,sans-Serif" font-size="10.00">MPR_EXECUTOR=lambda</text>
|
<text xml:space="preserve" text-anchor="middle" x="131.27" y="-529.4" font-family="Helvetica,sans-Serif" font-size="10.00">local</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- transcode -->
|
<!-- sfn_start -->
|
||||||
<g id="node12" class="node">
|
<g id="node16" class="node">
|
||||||
<title>transcode</title>
|
<title>sfn_start</title>
|
||||||
<path fill="none" stroke="black" d="M172.88,-513C172.88,-513 93.12,-513 93.12,-513 87.12,-513 81.12,-507 81.12,-501 81.12,-501 81.12,-482.5 81.12,-482.5 81.12,-476.5 87.12,-470.5 93.12,-470.5 93.12,-470.5 172.88,-470.5 172.88,-470.5 178.88,-470.5 184.88,-476.5 184.88,-482.5 184.88,-482.5 184.88,-501 184.88,-501 184.88,-507 178.88,-513 172.88,-513"/>
|
<path fill="none" stroke="black" d="M1525.88,-488C1525.88,-488 1428.12,-488 1428.12,-488 1422.12,-488 1416.12,-482 1416.12,-476 1416.12,-476 1416.12,-457.5 1416.12,-457.5 1416.12,-451.5 1422.12,-445.5 1428.12,-445.5 1428.12,-445.5 1525.88,-445.5 1525.88,-445.5 1531.88,-445.5 1537.88,-451.5 1537.88,-457.5 1537.88,-457.5 1537.88,-476 1537.88,-476 1537.88,-482 1531.88,-488 1525.88,-488"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="133" y="-495.7" font-family="Helvetica,sans-Serif" font-size="14.00">Transcode</text>
|
<text xml:space="preserve" text-anchor="middle" x="1477" y="-470.7" font-family="Helvetica,sans-Serif" font-size="14.00">Step Functions</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="133" y="-478.45" font-family="Helvetica,sans-Serif" font-size="14.00">(with preset)</text>
|
<text xml:space="preserve" text-anchor="middle" x="1477" y="-453.45" font-family="Helvetica,sans-Serif" font-size="14.00">start_execution</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- local->transcode -->
|
<!-- dispatch->sfn_start -->
|
||||||
<g id="edge13" class="edge">
|
|
||||||
<title>local->transcode</title>
|
|
||||||
<path fill="none" stroke="black" d="M209.38,-579C209.38,-547.27 209.38,-492 209.38,-492 209.38,-492 196.72,-492 196.72,-492"/>
|
|
||||||
<polygon fill="black" stroke="black" points="196.72,-488.5 186.72,-492 196.72,-495.5 196.72,-488.5"/>
|
|
||||||
</g>
|
|
||||||
<!-- trim -->
|
|
||||||
<g id="node13" class="node">
|
|
||||||
<title>trim</title>
|
|
||||||
<path fill="none" stroke="black" d="M372.5,-513C372.5,-513 239.5,-513 239.5,-513 233.5,-513 227.5,-507 227.5,-501 227.5,-501 227.5,-482.5 227.5,-482.5 227.5,-476.5 233.5,-470.5 239.5,-470.5 239.5,-470.5 372.5,-470.5 372.5,-470.5 378.5,-470.5 384.5,-476.5 384.5,-482.5 384.5,-482.5 384.5,-501 384.5,-501 384.5,-507 378.5,-513 372.5,-513"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="306" y="-495.7" font-family="Helvetica,sans-Serif" font-size="14.00">Trim</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="306" y="-478.45" font-family="Helvetica,sans-Serif" font-size="14.00">(-c:v copy -c:a copy)</text>
|
|
||||||
</g>
|
|
||||||
<!-- local->trim -->
|
|
||||||
<g id="edge14" class="edge">
|
|
||||||
<title>local->trim</title>
|
|
||||||
<path fill="none" stroke="black" d="M278.12,-579.22C278.12,-579.22 278.12,-524.75 278.12,-524.75"/>
|
|
||||||
<polygon fill="black" stroke="black" points="281.63,-524.75 278.13,-514.75 274.63,-524.75 281.63,-524.75"/>
|
|
||||||
</g>
|
|
||||||
<!-- progress -->
|
|
||||||
<g id="node17" class="node">
|
|
||||||
<title>progress</title>
|
|
||||||
<polygon fill="none" stroke="black" points="241.5,-407.25 84.5,-407.25 84.5,-364.75 247.5,-364.75 247.5,-401.25 241.5,-407.25"/>
|
|
||||||
<polyline fill="none" stroke="black" points="241.5,-407.25 241.5,-401.25"/>
|
|
||||||
<polyline fill="none" stroke="black" points="247.5,-401.25 241.5,-401.25"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="166" y="-389.95" font-family="Helvetica,sans-Serif" font-size="14.00">Progress Updates</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="166" y="-372.7" font-family="Helvetica,sans-Serif" font-size="14.00">(gRPC → Redis → DB)</text>
|
|
||||||
</g>
|
|
||||||
<!-- transcode->progress -->
|
|
||||||
<g id="edge15" class="edge">
|
|
||||||
<title>transcode->progress</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M134.69,-470.09C134.69,-470.09 134.69,-419.14 134.69,-419.14"/>
|
|
||||||
<polygon fill="black" stroke="black" points="138.19,-419.14 134.69,-409.14 131.19,-419.14 138.19,-419.14"/>
|
|
||||||
</g>
|
|
||||||
<!-- trim->progress -->
|
|
||||||
<g id="edge16" class="edge">
|
|
||||||
<title>trim->progress</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M237.5,-470.09C237.5,-470.09 237.5,-419.14 237.5,-419.14"/>
|
|
||||||
<polygon fill="black" stroke="black" points="241,-419.14 237.5,-409.14 234,-419.14 241,-419.14"/>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_stream -->
|
|
||||||
<g id="node14" class="node">
|
|
||||||
<title>grpc_stream</title>
|
|
||||||
<path fill="none" stroke="black" d="M304.33,-286.5C304.33,-286.5 89.19,-286.5 89.19,-286.5 83.19,-286.5 73.67,-281.64 70.15,-276.78 70.15,-276.78 22.71,-211.22 22.71,-211.22 19.19,-206.36 21.67,-201.5 27.67,-201.5 27.67,-201.5 242.81,-201.5 242.81,-201.5 248.81,-201.5 258.33,-206.36 261.85,-211.22 261.85,-211.22 309.29,-276.78 309.29,-276.78 312.81,-281.64 310.33,-286.5 304.33,-286.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="166" y="-247.95" font-family="Helvetica,sans-Serif" font-size="14.00">StreamProgress</text>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="166" y="-230.7" font-family="Helvetica,sans-Serif" font-size="14.00">(server streaming)</text>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_stream->processing -->
|
|
||||||
<g id="edge18" class="edge">
|
|
||||||
<title>grpc_stream->processing</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M166,-201.1C166,-167.71 166,-127 166,-127 166,-127 770.51,-127 770.51,-127"/>
|
|
||||||
<polygon fill="black" stroke="black" points="770.51,-130.5 780.51,-127 770.51,-123.5 770.51,-130.5"/>
|
|
||||||
<text xml:space="preserve" text-anchor="middle" x="476.38" y="-170" font-family="Helvetica,sans-Serif" font-size="10.00">update status</text>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_submit->pending -->
|
|
||||||
<g id="edge20" class="edge">
|
|
||||||
<title>grpc_submit->pending</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M450,-201.06C450,-186.11 450,-173 450,-173 450,-173 912.08,-173 912.08,-173 912.08,-173 912.08,-214.2 912.08,-214.2"/>
|
|
||||||
<polygon fill="black" stroke="black" points="908.58,-214.2 912.08,-224.2 915.58,-214.2 908.58,-214.2"/>
|
|
||||||
</g>
|
|
||||||
<!-- grpc_cancel->cancelled -->
|
|
||||||
<g id="edge22" class="edge">
|
|
||||||
<title>grpc_cancel->cancelled</title>
|
|
||||||
<path fill="none" stroke="black" stroke-dasharray="5,2" d="M728.29,-214C836.93,-214 1062.12,-214 1062.12,-214 1062.12,-214 1062.12,-63.76 1062.12,-63.76"/>
|
|
||||||
<polygon fill="black" stroke="black" points="1065.62,-63.76 1062.12,-53.76 1058.62,-63.76 1065.62,-63.76"/>
|
|
||||||
</g>
|
|
||||||
<!-- progress->grpc_stream -->
|
|
||||||
<g id="edge17" class="edge">
|
<g id="edge17" class="edge">
|
||||||
<title>progress->grpc_stream</title>
|
<title>dispatch->sfn_start</title>
|
||||||
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M166,-364.43C166,-364.43 166,-298.49 166,-298.49"/>
|
<path fill="none" stroke="black" d="M336.81,-552.63C336.81,-533.84 336.81,-467 336.81,-467 336.81,-467 1404.18,-467 1404.18,-467"/>
|
||||||
<polygon fill="black" stroke="black" points="169.5,-298.49 166,-288.49 162.5,-298.49 169.5,-298.49"/>
|
<polygon fill="black" stroke="black" points="1404.18,-470.5 1414.18,-467 1404.18,-463.5 1404.18,-470.5"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="204.62" y="-333.25" font-family="Helvetica,sans-Serif" font-size="10.00">stream to client</text>
|
<text xml:space="preserve" text-anchor="middle" x="809.3" y="-470.25" font-family="Helvetica,sans-Serif" font-size="10.00">lambda</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3_download -->
|
||||||
|
<g id="node12" class="node">
|
||||||
|
<title>s3_download</title>
|
||||||
|
<path fill="none" stroke="black" d="M144.38,-402.5C144.38,-402.5 61.62,-402.5 61.62,-402.5 55.62,-402.5 49.62,-396.5 49.62,-390.5 49.62,-390.5 49.62,-372 49.62,-372 49.62,-366 55.62,-360 61.62,-360 61.62,-360 144.38,-360 144.38,-360 150.38,-360 156.38,-366 156.38,-372 156.38,-372 156.38,-390.5 156.38,-390.5 156.38,-396.5 150.38,-402.5 144.38,-402.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="103" y="-385.2" font-family="Helvetica,sans-Serif" font-size="14.00">S3 Download</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="103" y="-367.95" font-family="Helvetica,sans-Serif" font-size="14.00">(MinIO)</text>
|
||||||
|
</g>
|
||||||
|
<!-- celery_task->s3_download -->
|
||||||
|
<g id="edge12" class="edge">
|
||||||
|
<title>celery_task->s3_download</title>
|
||||||
|
<path fill="none" stroke="black" d="M103,-445.17C103,-445.17 103,-414.33 103,-414.33"/>
|
||||||
|
<polygon fill="black" stroke="black" points="106.5,-414.33 103,-404.33 99.5,-414.33 106.5,-414.33"/>
|
||||||
|
</g>
|
||||||
|
<!-- ffmpeg_local -->
|
||||||
|
<g id="node13" class="node">
|
||||||
|
<title>ffmpeg_local</title>
|
||||||
|
<path fill="none" stroke="black" d="M153,-317C153,-317 59,-317 59,-317 53,-317 47,-311 47,-305 47,-305 47,-286.5 47,-286.5 47,-280.5 53,-274.5 59,-274.5 59,-274.5 153,-274.5 153,-274.5 159,-274.5 165,-280.5 165,-286.5 165,-286.5 165,-305 165,-305 165,-311 159,-317 153,-317"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="106" y="-299.7" font-family="Helvetica,sans-Serif" font-size="14.00">FFmpeg</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="106" y="-282.45" font-family="Helvetica,sans-Serif" font-size="14.00">transcode/trim</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3_download->ffmpeg_local -->
|
||||||
|
<g id="edge13" class="edge">
|
||||||
|
<title>s3_download->ffmpeg_local</title>
|
||||||
|
<path fill="none" stroke="black" d="M103,-359.67C103,-359.67 103,-328.83 103,-328.83"/>
|
||||||
|
<polygon fill="black" stroke="black" points="106.5,-328.83 103,-318.83 99.5,-328.83 106.5,-328.83"/>
|
||||||
|
</g>
|
||||||
|
<!-- s3_upload -->
|
||||||
|
<g id="node14" class="node">
|
||||||
|
<title>s3_upload</title>
|
||||||
|
<path fill="none" stroke="black" d="M138.62,-229.5C138.62,-229.5 75.38,-229.5 75.38,-229.5 69.38,-229.5 63.38,-223.5 63.38,-217.5 63.38,-217.5 63.38,-199 63.38,-199 63.38,-193 69.38,-187 75.38,-187 75.38,-187 138.62,-187 138.62,-187 144.62,-187 150.62,-193 150.62,-199 150.62,-199 150.62,-217.5 150.62,-217.5 150.62,-223.5 144.62,-229.5 138.62,-229.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="107" y="-212.2" font-family="Helvetica,sans-Serif" font-size="14.00">S3 Upload</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="107" y="-194.95" font-family="Helvetica,sans-Serif" font-size="14.00">(MinIO)</text>
|
||||||
|
</g>
|
||||||
|
<!-- ffmpeg_local->s3_upload -->
|
||||||
|
<g id="edge14" class="edge">
|
||||||
|
<title>ffmpeg_local->s3_upload</title>
|
||||||
|
<path fill="none" stroke="black" d="M107,-274.12C107,-274.12 107,-241.45 107,-241.45"/>
|
||||||
|
<polygon fill="black" stroke="black" points="110.5,-241.45 107,-231.45 103.5,-241.45 110.5,-241.45"/>
|
||||||
|
</g>
|
||||||
|
<!-- db_update -->
|
||||||
|
<g id="node15" class="node">
|
||||||
|
<title>db_update</title>
|
||||||
|
<path fill="none" stroke="black" d="M180.88,-144C180.88,-144 35.12,-144 35.12,-144 29.12,-144 23.12,-138 23.12,-132 23.12,-132 23.12,-113.5 23.12,-113.5 23.12,-107.5 29.12,-101.5 35.12,-101.5 35.12,-101.5 180.88,-101.5 180.88,-101.5 186.88,-101.5 192.88,-107.5 192.88,-113.5 192.88,-113.5 192.88,-132 192.88,-132 192.88,-138 186.88,-144 180.88,-144"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="108" y="-126.7" font-family="Helvetica,sans-Serif" font-size="14.00">DB Update</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="108" y="-109.45" font-family="Helvetica,sans-Serif" font-size="14.00">(update_job_progress)</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3_upload->db_update -->
|
||||||
|
<g id="edge15" class="edge">
|
||||||
|
<title>s3_upload->db_update</title>
|
||||||
|
<path fill="none" stroke="black" d="M107,-186.67C107,-186.67 107,-155.83 107,-155.83"/>
|
||||||
|
<polygon fill="black" stroke="black" points="110.5,-155.83 107,-145.83 103.5,-155.83 110.5,-155.83"/>
|
||||||
|
</g>
|
||||||
|
<!-- db_update->completed -->
|
||||||
|
<g id="edge16" class="edge">
|
||||||
|
<title>db_update->completed</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M193.17,-117C345.61,-117 649.29,-117 649.29,-117 649.29,-117 649.29,-43 649.29,-43 649.29,-43 675.4,-43 675.4,-43"/>
|
||||||
|
<polygon fill="black" stroke="black" points="675.4,-46.5 685.4,-43 675.4,-39.5 675.4,-46.5"/>
|
||||||
|
</g>
|
||||||
|
<!-- lambda_fn -->
|
||||||
|
<g id="node17" class="node">
|
||||||
|
<title>lambda_fn</title>
|
||||||
|
<path fill="none" stroke="black" d="M1546,-402.5C1546,-402.5 1428,-402.5 1428,-402.5 1422,-402.5 1416,-396.5 1416,-390.5 1416,-390.5 1416,-372 1416,-372 1416,-366 1422,-360 1428,-360 1428,-360 1546,-360 1546,-360 1552,-360 1558,-366 1558,-372 1558,-372 1558,-390.5 1558,-390.5 1558,-396.5 1552,-402.5 1546,-402.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1487" y="-385.2" font-family="Helvetica,sans-Serif" font-size="14.00">Lambda</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1487" y="-367.95" font-family="Helvetica,sans-Serif" font-size="14.00">FFmpeg container</text>
|
||||||
|
</g>
|
||||||
|
<!-- sfn_start->lambda_fn -->
|
||||||
|
<g id="edge18" class="edge">
|
||||||
|
<title>sfn_start->lambda_fn</title>
|
||||||
|
<path fill="none" stroke="black" d="M1477,-445.17C1477,-445.17 1477,-414.33 1477,-414.33"/>
|
||||||
|
<polygon fill="black" stroke="black" points="1480.5,-414.33 1477,-404.33 1473.5,-414.33 1480.5,-414.33"/>
|
||||||
|
</g>
|
||||||
|
<!-- s3_dl_aws -->
|
||||||
|
<g id="node18" class="node">
|
||||||
|
<title>s3_dl_aws</title>
|
||||||
|
<path fill="none" stroke="black" d="M1534.38,-317C1534.38,-317 1451.62,-317 1451.62,-317 1445.62,-317 1439.62,-311 1439.62,-305 1439.62,-305 1439.62,-286.5 1439.62,-286.5 1439.62,-280.5 1445.62,-274.5 1451.62,-274.5 1451.62,-274.5 1534.38,-274.5 1534.38,-274.5 1540.38,-274.5 1546.38,-280.5 1546.38,-286.5 1546.38,-286.5 1546.38,-305 1546.38,-305 1546.38,-311 1540.38,-317 1534.38,-317"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1493" y="-299.7" font-family="Helvetica,sans-Serif" font-size="14.00">S3 Download</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1493" y="-282.45" font-family="Helvetica,sans-Serif" font-size="14.00">(AWS)</text>
|
||||||
|
</g>
|
||||||
|
<!-- lambda_fn->s3_dl_aws -->
|
||||||
|
<g id="edge19" class="edge">
|
||||||
|
<title>lambda_fn->s3_dl_aws</title>
|
||||||
|
<path fill="none" stroke="black" d="M1493,-359.67C1493,-359.67 1493,-328.83 1493,-328.83"/>
|
||||||
|
<polygon fill="black" stroke="black" points="1496.5,-328.83 1493,-318.83 1489.5,-328.83 1496.5,-328.83"/>
|
||||||
|
</g>
|
||||||
|
<!-- ffmpeg_aws -->
|
||||||
|
<g id="node19" class="node">
|
||||||
|
<title>ffmpeg_aws</title>
|
||||||
|
<path fill="none" stroke="black" d="M1545,-229.5C1545,-229.5 1451,-229.5 1451,-229.5 1445,-229.5 1439,-223.5 1439,-217.5 1439,-217.5 1439,-199 1439,-199 1439,-193 1445,-187 1451,-187 1451,-187 1545,-187 1545,-187 1551,-187 1557,-193 1557,-199 1557,-199 1557,-217.5 1557,-217.5 1557,-223.5 1551,-229.5 1545,-229.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1498" y="-212.2" font-family="Helvetica,sans-Serif" font-size="14.00">FFmpeg</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1498" y="-194.95" font-family="Helvetica,sans-Serif" font-size="14.00">transcode/trim</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3_dl_aws->ffmpeg_aws -->
|
||||||
|
<g id="edge20" class="edge">
|
||||||
|
<title>s3_dl_aws->ffmpeg_aws</title>
|
||||||
|
<path fill="none" stroke="black" d="M1493,-274.12C1493,-274.12 1493,-241.45 1493,-241.45"/>
|
||||||
|
<polygon fill="black" stroke="black" points="1496.5,-241.45 1493,-231.45 1489.5,-241.45 1496.5,-241.45"/>
|
||||||
|
</g>
|
||||||
|
<!-- s3_ul_aws -->
|
||||||
|
<g id="node20" class="node">
|
||||||
|
<title>s3_ul_aws</title>
|
||||||
|
<path fill="none" stroke="black" d="M1532.62,-144C1532.62,-144 1469.38,-144 1469.38,-144 1463.38,-144 1457.38,-138 1457.38,-132 1457.38,-132 1457.38,-113.5 1457.38,-113.5 1457.38,-107.5 1463.38,-101.5 1469.38,-101.5 1469.38,-101.5 1532.62,-101.5 1532.62,-101.5 1538.62,-101.5 1544.62,-107.5 1544.62,-113.5 1544.62,-113.5 1544.62,-132 1544.62,-132 1544.62,-138 1538.62,-144 1532.62,-144"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1501" y="-126.7" font-family="Helvetica,sans-Serif" font-size="14.00">S3 Upload</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1501" y="-109.45" font-family="Helvetica,sans-Serif" font-size="14.00">(AWS)</text>
|
||||||
|
</g>
|
||||||
|
<!-- ffmpeg_aws->s3_ul_aws -->
|
||||||
|
<g id="edge21" class="edge">
|
||||||
|
<title>ffmpeg_aws->s3_ul_aws</title>
|
||||||
|
<path fill="none" stroke="black" d="M1501,-186.67C1501,-186.67 1501,-155.83 1501,-155.83"/>
|
||||||
|
<polygon fill="black" stroke="black" points="1504.5,-155.83 1501,-145.83 1497.5,-155.83 1504.5,-155.83"/>
|
||||||
|
</g>
|
||||||
|
<!-- callback -->
|
||||||
|
<g id="node21" class="node">
|
||||||
|
<title>callback</title>
|
||||||
|
<path fill="none" stroke="black" d="M1585.12,-58.5C1585.12,-58.5 1422.88,-58.5 1422.88,-58.5 1416.88,-58.5 1410.88,-52.5 1410.88,-46.5 1410.88,-46.5 1410.88,-28 1410.88,-28 1410.88,-22 1416.88,-16 1422.88,-16 1422.88,-16 1585.12,-16 1585.12,-16 1591.12,-16 1597.12,-22 1597.12,-28 1597.12,-28 1597.12,-46.5 1597.12,-46.5 1597.12,-52.5 1591.12,-58.5 1585.12,-58.5"/>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1504" y="-41.2" font-family="Helvetica,sans-Serif" font-size="14.00">HTTP Callback</text>
|
||||||
|
<text xml:space="preserve" text-anchor="middle" x="1504" y="-23.95" font-family="Helvetica,sans-Serif" font-size="14.00">POST /jobs/{id}/callback</text>
|
||||||
|
</g>
|
||||||
|
<!-- s3_ul_aws->callback -->
|
||||||
|
<g id="edge22" class="edge">
|
||||||
|
<title>s3_ul_aws->callback</title>
|
||||||
|
<path fill="none" stroke="black" d="M1501,-101.17C1501,-101.17 1501,-70.33 1501,-70.33"/>
|
||||||
|
<polygon fill="black" stroke="black" points="1504.5,-70.33 1501,-60.33 1497.5,-70.33 1504.5,-70.33"/>
|
||||||
|
</g>
|
||||||
|
<!-- callback->completed -->
|
||||||
|
<g id="edge23" class="edge">
|
||||||
|
<title>callback->completed</title>
|
||||||
|
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M1427.5,-58.88C1427.5,-69.48 1427.5,-80 1427.5,-80 1427.5,-80 786.08,-80 786.08,-80 786.08,-80 786.08,-67.14 786.08,-67.14"/>
|
||||||
|
<polygon fill="black" stroke="black" points="789.58,-67.14 786.08,-57.14 782.58,-67.14 789.58,-67.14"/>
|
||||||
</g>
|
</g>
|
||||||
</g>
|
</g>
|
||||||
</svg>
|
</svg>
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 21 KiB After Width: | Height: | Size: 24 KiB |
170
docs/architecture/04-media-storage.md
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
# Media Storage Architecture
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
MPR uses **S3-compatible storage** everywhere. Locally via MinIO, in production via AWS S3. The same boto3 code and S3 keys work in both environments - the only difference is the `S3_ENDPOINT_URL` env var.
|
||||||
|
|
||||||
|
## Storage Strategy
|
||||||
|
|
||||||
|
### S3 Buckets
|
||||||
|
|
||||||
|
| Bucket | Env Var | Purpose |
|
||||||
|
|--------|---------|---------|
|
||||||
|
| `mpr-media-in` | `S3_BUCKET_IN` | Source media files |
|
||||||
|
| `mpr-media-out` | `S3_BUCKET_OUT` | Transcoded/trimmed output |
|
||||||
|
|
||||||
|
### S3 Keys as File Paths
|
||||||
|
- **Database**: Stores S3 object keys (e.g., `video1.mp4`, `subfolder/video3.mp4`)
|
||||||
|
- **Local dev**: MinIO serves these via S3 API on port 9000
|
||||||
|
- **AWS**: Real S3, same keys, different endpoint
|
||||||
|
|
||||||
|
### Why S3 Everywhere?
|
||||||
|
1. **Identical code paths** - no branching between local and cloud
|
||||||
|
2. **Seamless executor switching** - Celery and Lambda both use boto3
|
||||||
|
3. **Cloud-native** - ready for production without refactoring
|
||||||
|
|
||||||
|
## Local Development (MinIO)
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
```bash
|
||||||
|
S3_ENDPOINT_URL=http://minio:9000
|
||||||
|
S3_BUCKET_IN=mpr-media-in
|
||||||
|
S3_BUCKET_OUT=mpr-media-out
|
||||||
|
AWS_ACCESS_KEY_ID=minioadmin
|
||||||
|
AWS_SECRET_ACCESS_KEY=minioadmin
|
||||||
|
```
|
||||||
|
|
||||||
|
### How It Works
|
||||||
|
- MinIO runs as a Docker container (port 9000 API, port 9001 console)
|
||||||
|
- `minio-init` container creates buckets and sets public read access on startup
|
||||||
|
- Nginx proxies `/media/in/` and `/media/out/` to MinIO buckets
|
||||||
|
- Upload files via MinIO Console (http://localhost:9001) or `mc` CLI
|
||||||
|
|
||||||
|
### Upload Files to MinIO
|
||||||
|
```bash
|
||||||
|
# Using mc CLI
|
||||||
|
mc alias set local http://localhost:9000 minioadmin minioadmin
|
||||||
|
mc cp video.mp4 local/mpr-media-in/
|
||||||
|
|
||||||
|
# Using aws CLI with endpoint override
|
||||||
|
aws --endpoint-url http://localhost:9000 s3 cp video.mp4 s3://mpr-media-in/
|
||||||
|
```
|
||||||
|
|
||||||
|
## AWS Production (S3)
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
```bash
|
||||||
|
# No S3_ENDPOINT_URL = uses real AWS S3
|
||||||
|
S3_BUCKET_IN=mpr-media-in
|
||||||
|
S3_BUCKET_OUT=mpr-media-out
|
||||||
|
AWS_REGION=us-east-1
|
||||||
|
AWS_ACCESS_KEY_ID=<real-key>
|
||||||
|
AWS_SECRET_ACCESS_KEY=<real-secret>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Upload Files to S3
|
||||||
|
```bash
|
||||||
|
aws s3 cp video.mp4 s3://mpr-media-in/
|
||||||
|
aws s3 sync /local/media/ s3://mpr-media-in/
|
||||||
|
```
|
||||||
|
|
||||||
|
## GCP Production (GCS via S3 compatibility)
|
||||||
|
|
||||||
|
GCS exposes an S3-compatible API. The same `core/storage/s3.py` boto3 code works
|
||||||
|
with no changes — only the endpoint and credentials differ.
|
||||||
|
|
||||||
|
### GCS HMAC Keys
|
||||||
|
Generate under **Cloud Storage → Settings → Interoperability** in the GCP console.
|
||||||
|
These act as `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`.
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
```bash
|
||||||
|
S3_ENDPOINT_URL=https://storage.googleapis.com
|
||||||
|
S3_BUCKET_IN=mpr-media-in
|
||||||
|
S3_BUCKET_OUT=mpr-media-out
|
||||||
|
AWS_ACCESS_KEY_ID=<GCS HMAC access key>
|
||||||
|
AWS_SECRET_ACCESS_KEY=<GCS HMAC secret>
|
||||||
|
|
||||||
|
# Executor
|
||||||
|
MPR_EXECUTOR=gcp
|
||||||
|
GCP_PROJECT_ID=my-project
|
||||||
|
GCP_REGION=us-central1
|
||||||
|
CLOUD_RUN_JOB=mpr-transcode
|
||||||
|
CALLBACK_URL=https://mpr.mcrn.ar/api
|
||||||
|
CALLBACK_API_KEY=<secret>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Upload Files to GCS
|
||||||
|
```bash
|
||||||
|
gcloud storage cp video.mp4 gs://mpr-media-in/
|
||||||
|
|
||||||
|
# Or with the aws CLI via compat endpoint
|
||||||
|
aws --endpoint-url https://storage.googleapis.com s3 cp video.mp4 s3://mpr-media-in/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cloud Run Job Handler
|
||||||
|
`core/task/gcp_handler.py` is the Cloud Run Job entrypoint. It reads the job payload
|
||||||
|
from `MPR_JOB_PAYLOAD` (injected by `GCPExecutor`), uses `core/storage` for all
|
||||||
|
GCS access (S3 compat), and POSTs the completion callback to the API.
|
||||||
|
|
||||||
|
Set the Cloud Run Job command to: `python -m core.task.gcp_handler`
|
||||||
|
|
||||||
|
## Storage Module
|
||||||
|
|
||||||
|
`core/storage/` package provides all S3 operations:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from core.storage import (
|
||||||
|
get_s3_client, # boto3 client (MinIO or AWS)
|
||||||
|
list_objects, # List bucket contents, filter by extension
|
||||||
|
download_file, # Download S3 object to local path
|
||||||
|
download_to_temp, # Download to temp file (caller cleans up)
|
||||||
|
upload_file, # Upload local file to S3
|
||||||
|
get_presigned_url, # Generate presigned URL
|
||||||
|
BUCKET_IN, # Input bucket name
|
||||||
|
BUCKET_OUT, # Output bucket name
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
## API Endpoints
|
||||||
|
|
||||||
|
### Scan Media (REST)
|
||||||
|
```http
|
||||||
|
POST /api/assets/scan
|
||||||
|
```
|
||||||
|
Lists objects in `S3_BUCKET_IN`, registers new media files.
|
||||||
|
|
||||||
|
### Scan Media (GraphQL)
|
||||||
|
```graphql
|
||||||
|
mutation { scanMediaFolder { found registered skipped files } }
|
||||||
|
```
|
||||||
|
|
||||||
|
## Job Flow with S3
|
||||||
|
|
||||||
|
### Local Mode (Celery)
|
||||||
|
1. Celery task receives `source_key` and `output_key`
|
||||||
|
2. Downloads source from `S3_BUCKET_IN` to temp file
|
||||||
|
3. Runs FFmpeg locally
|
||||||
|
4. Uploads result to `S3_BUCKET_OUT`
|
||||||
|
5. Cleans up temp files
|
||||||
|
|
||||||
|
### Lambda Mode (AWS)
|
||||||
|
1. Step Functions invokes Lambda with S3 keys
|
||||||
|
2. Lambda downloads source from `S3_BUCKET_IN` to `/tmp`
|
||||||
|
3. Runs FFmpeg in container
|
||||||
|
4. Uploads result to `S3_BUCKET_OUT`
|
||||||
|
5. Calls back to API with result
|
||||||
|
|
||||||
|
### Cloud Run Job Mode (GCP)
|
||||||
|
1. `GCPExecutor` triggers Cloud Run Job with payload in `MPR_JOB_PAYLOAD`
|
||||||
|
2. `core/task/gcp_handler.py` downloads source from `S3_BUCKET_IN` (GCS S3 compat)
|
||||||
|
3. Runs FFmpeg in container
|
||||||
|
4. Uploads result to `S3_BUCKET_OUT` (GCS S3 compat)
|
||||||
|
5. Calls back to API with result
|
||||||
|
|
||||||
|
All three paths use the same S3-compatible bucket names and key structure.
|
||||||
|
|
||||||
|
## Supported File Types
|
||||||
|
|
||||||
|
**Video:** `.mp4`, `.mkv`, `.avi`, `.mov`, `.webm`, `.flv`, `.wmv`, `.m4v`
|
||||||
|
**Audio:** `.mp3`, `.wav`, `.flac`, `.aac`, `.ogg`, `.m4a`
|
||||||
@@ -1,101 +1,212 @@
|
|||||||
<!DOCTYPE html>
|
<!doctype html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>MPR - Architecture</title>
|
<title>MPR - Architecture</title>
|
||||||
<link rel="stylesheet" href="styles.css">
|
<link rel="stylesheet" href="styles.css" />
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<h1>MPR - Media Processor</h1>
|
<h1>MPR - Media Processor</h1>
|
||||||
<p>A web-based media transcoding tool with professional architecture.</p>
|
<p>
|
||||||
|
Media transcoding platform with dual execution modes: local (Celery
|
||||||
|
+ MinIO) and cloud (AWS Step Functions + Lambda + S3).
|
||||||
|
</p>
|
||||||
|
|
||||||
<nav>
|
<nav>
|
||||||
<a href="#overview">System Overview</a>
|
<a href="#overview">System Overview</a>
|
||||||
<a href="#data-model">Data Model</a>
|
<a href="#data-model">Data Model</a>
|
||||||
<a href="#job-flow">Job Flow</a>
|
<a href="#job-flow">Job Flow</a>
|
||||||
</nav>
|
<a href="#media-storage">Media Storage</a>
|
||||||
|
</nav>
|
||||||
|
|
||||||
<h2 id="overview">System Overview</h2>
|
<h2 id="overview">System Overview</h2>
|
||||||
<div class="diagram-container">
|
<div class="diagram-container">
|
||||||
<div class="diagram">
|
<div class="diagram">
|
||||||
<h3>Architecture</h3>
|
<h3>Local Architecture (Development)</h3>
|
||||||
<object type="image/svg+xml" data="01-system-overview.svg">
|
<object type="image/svg+xml" data="01a-local-architecture.svg">
|
||||||
<img src="01-system-overview.svg" alt="System Overview">
|
<img
|
||||||
</object>
|
src="01a-local-architecture.svg"
|
||||||
<a href="01-system-overview.svg" target="_blank">Open full size</a>
|
alt="Local Architecture"
|
||||||
|
/>
|
||||||
|
</object>
|
||||||
|
<a href="01a-local-architecture.svg" target="_blank"
|
||||||
|
>Open full size</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="diagram">
|
||||||
|
<h3>AWS Architecture (Production)</h3>
|
||||||
|
<object type="image/svg+xml" data="01b-aws-architecture.svg">
|
||||||
|
<img
|
||||||
|
src="01b-aws-architecture.svg"
|
||||||
|
alt="AWS Architecture"
|
||||||
|
/>
|
||||||
|
</object>
|
||||||
|
<a href="01b-aws-architecture.svg" target="_blank"
|
||||||
|
>Open full size</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="legend">
|
<div class="legend">
|
||||||
<h3>Components</h3>
|
<h3>Components</h3>
|
||||||
<ul>
|
<ul>
|
||||||
<li><span class="color-box" style="background: #e8f4f8"></span> Reverse Proxy (nginx)</li>
|
<li>
|
||||||
<li><span class="color-box" style="background: #f0f8e8"></span> Application Layer (Django, FastAPI, UI)</li>
|
<span class="color-box" style="background: #e8f4f8"></span>
|
||||||
<li><span class="color-box" style="background: #fff8e8"></span> Worker Layer (Celery, Lambda)</li>
|
Reverse Proxy (nginx)
|
||||||
<li><span class="color-box" style="background: #f8e8f0"></span> Data Layer (PostgreSQL, Redis, SQS)</li>
|
</li>
|
||||||
<li><span class="color-box" style="background: #f0f0f0"></span> Storage (Local FS, S3)</li>
|
<li>
|
||||||
</ul>
|
<span class="color-box" style="background: #f0f8e8"></span>
|
||||||
</div>
|
Application Layer (Django Admin, GraphQL API, Timeline UI)
|
||||||
|
</li>
|
||||||
<h2 id="data-model">Data Model</h2>
|
<li>
|
||||||
<div class="diagram-container">
|
<span class="color-box" style="background: #fff8e8"></span>
|
||||||
<div class="diagram">
|
Worker Layer (Celery local mode)
|
||||||
<h3>Entity Relationships</h3>
|
</li>
|
||||||
<object type="image/svg+xml" data="02-data-model.svg">
|
<li>
|
||||||
<img src="02-data-model.svg" alt="Data Model">
|
<span class="color-box" style="background: #fde8d0"></span>
|
||||||
</object>
|
AWS (Step Functions, Lambda - cloud mode)
|
||||||
<a href="02-data-model.svg" target="_blank">Open full size</a>
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #f8e8f0"></span>
|
||||||
|
Data Layer (PostgreSQL, Redis)
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #f0f0f0"></span>
|
||||||
|
S3 Storage (MinIO local / AWS S3 cloud)
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="legend">
|
<h2 id="data-model">Data Model</h2>
|
||||||
<h3>Entities</h3>
|
<div class="diagram-container">
|
||||||
<ul>
|
<div class="diagram">
|
||||||
<li><span class="color-box" style="background: #4a90d9"></span> MediaAsset - Video/audio files with metadata</li>
|
<h3>Entity Relationships</h3>
|
||||||
<li><span class="color-box" style="background: #50b050"></span> TranscodePreset - Encoding configurations</li>
|
<object type="image/svg+xml" data="02-data-model.svg">
|
||||||
<li><span class="color-box" style="background: #d9534f"></span> TranscodeJob - Processing queue items</li>
|
<img src="02-data-model.svg" alt="Data Model" />
|
||||||
</ul>
|
</object>
|
||||||
</div>
|
<a href="02-data-model.svg" target="_blank">Open full size</a>
|
||||||
|
</div>
|
||||||
<h2 id="job-flow">Job Flow</h2>
|
|
||||||
<div class="diagram-container">
|
|
||||||
<div class="diagram">
|
|
||||||
<h3>Job Lifecycle</h3>
|
|
||||||
<object type="image/svg+xml" data="03-job-flow.svg">
|
|
||||||
<img src="03-job-flow.svg" alt="Job Flow">
|
|
||||||
</object>
|
|
||||||
<a href="03-job-flow.svg" target="_blank">Open full size</a>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="legend">
|
<div class="legend">
|
||||||
<h3>Job States</h3>
|
<h3>Entities</h3>
|
||||||
<ul>
|
<ul>
|
||||||
<li><span class="color-box" style="background: #ffc107"></span> PENDING - Waiting in queue</li>
|
<li>
|
||||||
<li><span class="color-box" style="background: #17a2b8"></span> PROCESSING - Worker executing</li>
|
<span class="color-box" style="background: #4a90d9"></span>
|
||||||
<li><span class="color-box" style="background: #28a745"></span> COMPLETED - Success</li>
|
MediaAsset - Video/audio files (S3 keys as paths)
|
||||||
<li><span class="color-box" style="background: #dc3545"></span> FAILED - Error occurred</li>
|
</li>
|
||||||
<li><span class="color-box" style="background: #6c757d"></span> CANCELLED - User cancelled</li>
|
<li>
|
||||||
</ul>
|
<span class="color-box" style="background: #50b050"></span>
|
||||||
</div>
|
TranscodePreset - Encoding configurations
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #d9534f"></span>
|
||||||
|
TranscodeJob - Processing queue (celery_task_id or
|
||||||
|
execution_arn)
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
<h2>Quick Reference</h2>
|
<h2 id="job-flow">Job Flow</h2>
|
||||||
<pre><code># Generate SVGs from DOT files
|
<div class="diagram-container">
|
||||||
dot -Tsvg 01-system-overview.dot -o 01-system-overview.svg
|
<div class="diagram">
|
||||||
dot -Tsvg 02-data-model.dot -o 02-data-model.svg
|
<h3>Job Lifecycle</h3>
|
||||||
dot -Tsvg 03-job-flow.dot -o 03-job-flow.svg
|
<object type="image/svg+xml" data="03-job-flow.svg">
|
||||||
|
<img src="03-job-flow.svg" alt="Job Flow" />
|
||||||
|
</object>
|
||||||
|
<a href="03-job-flow.svg" target="_blank">Open full size</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
# Or generate all at once
|
<div class="legend">
|
||||||
for f in *.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done</code></pre>
|
<h3>Job States</h3>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #ffc107"></span>
|
||||||
|
PENDING - Waiting in queue
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #17a2b8"></span>
|
||||||
|
PROCESSING - Worker executing
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #28a745"></span>
|
||||||
|
COMPLETED - Success
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #dc3545"></span>
|
||||||
|
FAILED - Error occurred
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #6c757d"></span>
|
||||||
|
CANCELLED - User cancelled
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<h3>Execution Modes</h3>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #e8f4e8"></span>
|
||||||
|
Local: Celery + MinIO (S3 API) + FFmpeg
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #fde8d0"></span>
|
||||||
|
Lambda: Step Functions + Lambda + AWS S3
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
<h2>Access Points</h2>
|
<h2 id="media-storage">Media Storage</h2>
|
||||||
<pre><code># Add to /etc/hosts
|
<div class="diagram-container">
|
||||||
|
<p>
|
||||||
|
MPR separates media into input and output paths for flexible
|
||||||
|
storage configuration.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<a href="04-media-storage.md" target="_blank"
|
||||||
|
>View Media Storage Documentation →</a
|
||||||
|
>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>API (GraphQL)</h2>
|
||||||
|
<pre><code># GraphiQL IDE
|
||||||
|
http://mpr.local.ar/graphql
|
||||||
|
|
||||||
|
# Queries
|
||||||
|
query { assets(status: "ready") { id filename duration } }
|
||||||
|
query { jobs(status: "processing") { id status progress } }
|
||||||
|
query { presets { id name container videoCodec } }
|
||||||
|
query { systemStatus { status version } }
|
||||||
|
|
||||||
|
# Mutations
|
||||||
|
mutation { scanMediaFolder { found registered skipped } }
|
||||||
|
mutation { createJob(input: { sourceAssetId: "...", presetId: "..." }) { id status } }
|
||||||
|
mutation { cancelJob(id: "...") { id status } }
|
||||||
|
mutation { retryJob(id: "...") { id status } }
|
||||||
|
mutation { updateAsset(id: "...", input: { comments: "..." }) { id comments } }
|
||||||
|
mutation { deleteAsset(id: "...") { ok } }
|
||||||
|
|
||||||
|
# Lambda callback (REST)
|
||||||
|
POST /api/jobs/{id}/callback - Lambda completion webhook</code></pre>
|
||||||
|
|
||||||
|
<h2>Access Points</h2>
|
||||||
|
<pre><code># Local development
|
||||||
127.0.0.1 mpr.local.ar
|
127.0.0.1 mpr.local.ar
|
||||||
|
http://mpr.local.ar/admin - Django Admin
|
||||||
|
http://mpr.local.ar/graphql - GraphiQL
|
||||||
|
http://mpr.local.ar/ - Timeline UI
|
||||||
|
http://localhost:9001 - MinIO Console
|
||||||
|
|
||||||
# URLs
|
# AWS deployment
|
||||||
http://mpr.local.ar/admin - Django Admin
|
https://mpr.mcrn.ar/ - Production</code></pre>
|
||||||
http://mpr.local.ar/api - FastAPI (docs at /api/docs)
|
|
||||||
http://mpr.local.ar/ui - Timeline UI</code></pre>
|
<h2>Quick Reference</h2>
|
||||||
</body>
|
<pre><code># Render SVGs from DOT files
|
||||||
|
for f in *.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done
|
||||||
|
|
||||||
|
# Switch executor mode
|
||||||
|
MPR_EXECUTOR=local # Celery + MinIO
|
||||||
|
MPR_EXECUTOR=lambda # Step Functions + Lambda + S3</code></pre>
|
||||||
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
279
docs/index.html
Normal file
@@ -0,0 +1,279 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>MPR - Architecture</title>
|
||||||
|
<link rel="stylesheet" href="architecture/styles.css" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>MPR - Media Processor</h1>
|
||||||
|
<p>
|
||||||
|
Media transcoding platform with three execution modes: local (Celery
|
||||||
|
+ MinIO), AWS (Step Functions + Lambda + S3), and GCP (Cloud Run
|
||||||
|
Jobs + GCS). Storage is S3-compatible across all environments.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<nav>
|
||||||
|
<a href="#overview">System Overview</a>
|
||||||
|
<a href="#data-model">Data Model</a>
|
||||||
|
<a href="#job-flow">Job Flow</a>
|
||||||
|
<a href="#media-storage">Media Storage</a>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<h2 id="overview">System Overview</h2>
|
||||||
|
<div class="diagram-container">
|
||||||
|
<div class="diagram">
|
||||||
|
<h3>Local Architecture (Development)</h3>
|
||||||
|
<object
|
||||||
|
type="image/svg+xml"
|
||||||
|
data="architecture/01a-local-architecture.svg"
|
||||||
|
>
|
||||||
|
<img
|
||||||
|
src="architecture/01a-local-architecture.svg"
|
||||||
|
alt="Local Architecture"
|
||||||
|
/>
|
||||||
|
</object>
|
||||||
|
<a
|
||||||
|
href="architecture/01a-local-architecture.svg"
|
||||||
|
target="_blank"
|
||||||
|
>Open full size</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="diagram">
|
||||||
|
<h3>AWS Architecture (Production)</h3>
|
||||||
|
<object
|
||||||
|
type="image/svg+xml"
|
||||||
|
data="architecture/01b-aws-architecture.svg"
|
||||||
|
>
|
||||||
|
<img
|
||||||
|
src="architecture/01b-aws-architecture.svg"
|
||||||
|
alt="AWS Architecture"
|
||||||
|
/>
|
||||||
|
</object>
|
||||||
|
<a href="architecture/01b-aws-architecture.svg" target="_blank"
|
||||||
|
>Open full size</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="diagram">
|
||||||
|
<h3>GCP Architecture (Production)</h3>
|
||||||
|
<object
|
||||||
|
type="image/svg+xml"
|
||||||
|
data="architecture/01c-gcp-architecture.svg"
|
||||||
|
>
|
||||||
|
<img
|
||||||
|
src="architecture/01c-gcp-architecture.svg"
|
||||||
|
alt="GCP Architecture"
|
||||||
|
/>
|
||||||
|
</object>
|
||||||
|
<a href="architecture/01c-gcp-architecture.svg" target="_blank"
|
||||||
|
>Open full size</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="legend">
|
||||||
|
<h3>Components</h3>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #e8f4f8"></span>
|
||||||
|
Reverse Proxy (nginx)
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #f0f8e8"></span>
|
||||||
|
Application Layer (Django Admin, GraphQL API, Timeline UI)
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #fff8e8"></span>
|
||||||
|
Worker Layer (Celery local mode)
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #fde8d0"></span>
|
||||||
|
AWS (Step Functions, Lambda)
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #e8f0fd"></span>
|
||||||
|
GCP (Cloud Run Jobs + GCS)
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #f8e8f0"></span>
|
||||||
|
Data Layer (PostgreSQL, Redis)
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #f0f0f0"></span>
|
||||||
|
S3-compatible Storage (MinIO / AWS S3 / GCS)
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2 id="data-model">Data Model</h2>
|
||||||
|
<div class="diagram-container">
|
||||||
|
<div class="diagram">
|
||||||
|
<h3>Entity Relationships</h3>
|
||||||
|
<object
|
||||||
|
type="image/svg+xml"
|
||||||
|
data="architecture/02-data-model.svg"
|
||||||
|
>
|
||||||
|
<img
|
||||||
|
src="architecture/02-data-model.svg"
|
||||||
|
alt="Data Model"
|
||||||
|
/>
|
||||||
|
</object>
|
||||||
|
<a href="architecture/02-data-model.svg" target="_blank"
|
||||||
|
>Open full size</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="legend">
|
||||||
|
<h3>Entities</h3>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #4a90d9"></span>
|
||||||
|
MediaAsset - Video/audio files with metadata
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #50b050"></span>
|
||||||
|
TranscodePreset - Encoding configurations
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #d9534f"></span>
|
||||||
|
TranscodeJob - Processing queue items
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2 id="job-flow">Job Flow</h2>
|
||||||
|
<div class="diagram-container">
|
||||||
|
<div class="diagram">
|
||||||
|
<h3>Job Lifecycle</h3>
|
||||||
|
<object
|
||||||
|
type="image/svg+xml"
|
||||||
|
data="architecture/03-job-flow.svg"
|
||||||
|
>
|
||||||
|
<img src="architecture/03-job-flow.svg" alt="Job Flow" />
|
||||||
|
</object>
|
||||||
|
<a href="architecture/03-job-flow.svg" target="_blank"
|
||||||
|
>Open full size</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="legend">
|
||||||
|
<h3>Job States</h3>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #ffc107"></span>
|
||||||
|
PENDING - Waiting in queue
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #17a2b8"></span>
|
||||||
|
PROCESSING - Worker executing
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #28a745"></span>
|
||||||
|
COMPLETED - Success
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #dc3545"></span>
|
||||||
|
FAILED - Error occurred
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #6c757d"></span>
|
||||||
|
CANCELLED - User cancelled
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2 id="media-storage">Media Storage</h2>
|
||||||
|
<div class="diagram-container">
|
||||||
|
<p>
|
||||||
|
MPR separates media into <strong>input</strong> and
|
||||||
|
<strong>output</strong> paths, each independently configurable.
|
||||||
|
File paths are stored
|
||||||
|
<strong>relative to their respective root</strong> to ensure
|
||||||
|
portability between local development and cloud deployments (AWS
|
||||||
|
S3, etc.).
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="legend">
|
||||||
|
<h3>Input / Output Separation</h3>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #4a90d9"></span>
|
||||||
|
<code>MEDIA_IN</code> - Source media files to process
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<span class="color-box" style="background: #50b050"></span>
|
||||||
|
<code>MEDIA_OUT</code> - Transcoded/trimmed output files
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<p><strong>Why Relative Paths?</strong></p>
|
||||||
|
<ul>
|
||||||
|
<li>Portability: Same database works locally and in cloud</li>
|
||||||
|
<li>Flexibility: Easy to switch between storage backends</li>
|
||||||
|
<li>Simplicity: No need to update paths when migrating</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="legend">
|
||||||
|
<h3>Local Development</h3>
|
||||||
|
<pre><code>MEDIA_IN=/app/media/in
|
||||||
|
MEDIA_OUT=/app/media/out
|
||||||
|
|
||||||
|
/app/media/
|
||||||
|
├── in/ # Source files
|
||||||
|
│ ├── video1.mp4
|
||||||
|
│ └── subfolder/video3.mp4
|
||||||
|
└── out/ # Transcoded output
|
||||||
|
└── video1_h264.mp4</code></pre>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="legend">
|
||||||
|
<h3>AWS/Cloud Deployment</h3>
|
||||||
|
<pre><code>MEDIA_IN=s3://source-bucket/media/
|
||||||
|
MEDIA_OUT=s3://output-bucket/transcoded/
|
||||||
|
MEDIA_BASE_URL=https://source-bucket.s3.amazonaws.com/media/</code></pre>
|
||||||
|
<p>
|
||||||
|
Database paths remain unchanged (already relative). Just upload
|
||||||
|
files to S3 and update environment variables.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="legend">
|
||||||
|
<h3>API (GraphQL)</h3>
|
||||||
|
<p>
|
||||||
|
All client interactions go through GraphQL at
|
||||||
|
<code>/graphql</code>.
|
||||||
|
</p>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<code>scanMediaFolder</code> - Scan S3 bucket for media
|
||||||
|
files
|
||||||
|
</li>
|
||||||
|
<li><code>createJob</code> - Create transcode/trim job</li>
|
||||||
|
<li>
|
||||||
|
<code>cancelJob / retryJob</code> - Job lifecycle management
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<code>updateAsset / deleteAsset</code> - Asset management
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<p><strong>Supported File Types:</strong></p>
|
||||||
|
<p>
|
||||||
|
Video: mp4, mkv, avi, mov, webm, flv, wmv, m4v<br />
|
||||||
|
Audio: mp3, wav, flac, aac, ogg, m4a
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Access Points</h2>
|
||||||
|
<pre><code># Add to /etc/hosts
|
||||||
|
127.0.0.1 mpr.local.ar
|
||||||
|
|
||||||
|
# URLs
|
||||||
|
http://mpr.local.ar/admin - Django Admin
|
||||||
|
http://mpr.local.ar/graphql - GraphiQL IDE
|
||||||
|
http://mpr.local.ar/ - Timeline UI</code></pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
125
docs/media-storage.html
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
<h1>Media Storage Architecture</h1>
|
||||||
|
<h2>Overview</h2>
|
||||||
|
<p>MPR separates media into <strong>input</strong> and <strong>output</strong> paths, each independently configurable. File paths are stored <strong>relative to their respective root</strong> to ensure portability between local development and cloud deployments (AWS S3, etc.).</p>
|
||||||
|
<h2>Storage Strategy</h2>
|
||||||
|
<h3>Input / Output Separation</h3>
|
||||||
|
<p>| Path | Env Var | Purpose |
|
||||||
|
|------|---------|---------|
|
||||||
|
| <code>MEDIA_IN</code> | <code>/app/media/in</code> | Source media files to process |
|
||||||
|
| <code>MEDIA_OUT</code> | <code>/app/media/out</code> | Transcoded/trimmed output files |</p>
|
||||||
|
<p>These can point to different locations or even different servers/buckets in production.</p>
|
||||||
|
<h3>File Path Storage</h3>
|
||||||
|
<ul>
|
||||||
|
<li><strong>Database</strong>: Stores only the relative path (e.g., <code>videos/sample.mp4</code>)</li>
|
||||||
|
<li><strong>Input Root</strong>: Configurable via <code>MEDIA_IN</code> env var</li>
|
||||||
|
<li><strong>Output Root</strong>: Configurable via <code>MEDIA_OUT</code> env var</li>
|
||||||
|
<li><strong>Serving</strong>: Base URL configurable via <code>MEDIA_BASE_URL</code> env var</li>
|
||||||
|
</ul>
|
||||||
|
<h3>Why Relative Paths?</h3>
|
||||||
|
<ol>
|
||||||
|
<li><strong>Portability</strong>: Same database works locally and in cloud</li>
|
||||||
|
<li><strong>Flexibility</strong>: Easy to switch between storage backends</li>
|
||||||
|
<li><strong>Simplicity</strong>: No need to update paths when migrating</li>
|
||||||
|
</ol>
|
||||||
|
<h2>Local Development</h2>
|
||||||
|
<h3>Configuration</h3>
|
||||||
|
<p><code>bash
|
||||||
|
MEDIA_IN=/app/media/in
|
||||||
|
MEDIA_OUT=/app/media/out</code></p>
|
||||||
|
<h3>File Structure</h3>
|
||||||
|
<p><code>/app/media/
|
||||||
|
├── in/ # Source files
|
||||||
|
│ ├── video1.mp4
|
||||||
|
│ ├── video2.mp4
|
||||||
|
│ └── subfolder/
|
||||||
|
│ └── video3.mp4
|
||||||
|
└── out/ # Transcoded output
|
||||||
|
├── video1_h264.mp4
|
||||||
|
└── video2_trimmed.mp4</code></p>
|
||||||
|
<h3>Database Storage</h3>
|
||||||
|
<p>```</p>
|
||||||
|
<h1>Source assets (scanned from media/in)</h1>
|
||||||
|
<p>filename: video1.mp4
|
||||||
|
file_path: video1.mp4</p>
|
||||||
|
<p>filename: video3.mp4
|
||||||
|
file_path: subfolder/video3.mp4
|
||||||
|
```</p>
|
||||||
|
<h3>URL Serving</h3>
|
||||||
|
<ul>
|
||||||
|
<li>Nginx serves input via <code>location /media/in { alias /app/media/in; }</code></li>
|
||||||
|
<li>Nginx serves output via <code>location /media/out { alias /app/media/out; }</code></li>
|
||||||
|
<li>Frontend accesses: <code>http://mpr.local.ar/media/in/video1.mp4</code></li>
|
||||||
|
<li>Video player: <code><video src="/media/in/video1.mp4" /></code></li>
|
||||||
|
</ul>
|
||||||
|
<h2>AWS/Cloud Deployment</h2>
|
||||||
|
<h3>S3 Configuration</h3>
|
||||||
|
<p>```bash</p>
|
||||||
|
<h1>Input and output can be different buckets/paths</h1>
|
||||||
|
<p>MEDIA_IN=s3://source-bucket/media/
|
||||||
|
MEDIA_OUT=s3://output-bucket/transcoded/
|
||||||
|
MEDIA_BASE_URL=https://source-bucket.s3.amazonaws.com/media/
|
||||||
|
```</p>
|
||||||
|
<h3>S3 Structure</h3>
|
||||||
|
<p>```
|
||||||
|
s3://source-bucket/media/
|
||||||
|
├── video1.mp4
|
||||||
|
└── subfolder/
|
||||||
|
└── video3.mp4</p>
|
||||||
|
<p>s3://output-bucket/transcoded/
|
||||||
|
├── video1_h264.mp4
|
||||||
|
└── video2_trimmed.mp4
|
||||||
|
```</p>
|
||||||
|
<h3>Database Storage (Same!)</h3>
|
||||||
|
<p>```
|
||||||
|
filename: video1.mp4
|
||||||
|
file_path: video1.mp4</p>
|
||||||
|
<p>filename: video3.mp4
|
||||||
|
file_path: subfolder/video3.mp4
|
||||||
|
```</p>
|
||||||
|
<h2>API Endpoints</h2>
|
||||||
|
<h3>Scan Media Folder</h3>
|
||||||
|
<p><code>http
|
||||||
|
POST /api/assets/scan</code></p>
|
||||||
|
<p><strong>Behavior:</strong>
|
||||||
|
1. Recursively scans <code>MEDIA_IN</code> directory
|
||||||
|
2. Finds all video/audio files (mp4, mkv, avi, mov, mp3, wav, etc.)
|
||||||
|
3. Stores paths <strong>relative to MEDIA_IN</strong>
|
||||||
|
4. Skips already-registered files (by filename)
|
||||||
|
5. Returns summary: <code>{ found, registered, skipped, files }</code></p>
|
||||||
|
<h3>Create Job</h3>
|
||||||
|
<p>```http
|
||||||
|
POST /api/jobs/
|
||||||
|
Content-Type: application/json</p>
|
||||||
|
<p>{
|
||||||
|
"source_asset_id": "uuid",
|
||||||
|
"preset_id": "uuid",
|
||||||
|
"trim_start": 10.0,
|
||||||
|
"trim_end": 30.0
|
||||||
|
}
|
||||||
|
```</p>
|
||||||
|
<p><strong>Behavior:</strong>
|
||||||
|
- Server sets <code>output_path</code> using <code>MEDIA_OUT</code> + generated filename
|
||||||
|
- Output goes to the output directory, not alongside source files</p>
|
||||||
|
<h2>Migration Guide</h2>
|
||||||
|
<h3>Moving from Local to S3</h3>
|
||||||
|
<ol>
|
||||||
|
<li>
|
||||||
|
<p><strong>Upload source files to S3:</strong>
|
||||||
|
<code>bash
|
||||||
|
aws s3 sync /app/media/in/ s3://source-bucket/media/
|
||||||
|
aws s3 sync /app/media/out/ s3://output-bucket/transcoded/</code></p>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<p><strong>Update environment variables:</strong>
|
||||||
|
<code>bash
|
||||||
|
MEDIA_IN=s3://source-bucket/media/
|
||||||
|
MEDIA_OUT=s3://output-bucket/transcoded/
|
||||||
|
MEDIA_BASE_URL=https://source-bucket.s3.amazonaws.com/media/</code></p>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<p><strong>Database paths remain unchanged</strong> (already relative)</p>
|
||||||
|
</li>
|
||||||
|
</ol>
|
||||||
|
<h2>Supported File Types</h2>
|
||||||
|
<p><strong>Video:</strong> <code>.mp4</code>, <code>.mkv</code>, <code>.avi</code>, <code>.mov</code>, <code>.webm</code>, <code>.flv</code>, <code>.wmv</code>, <code>.m4v</code>
|
||||||
|
<strong>Audio:</strong> <code>.mp3</code>, <code>.wav</code>, <code>.flac</code>, <code>.aac</code>, <code>.ogg</code>, <code>.m4a</code></p>
|
||||||
0
media/in/.gitkeep
Normal file
0
media/out/.gitkeep
Normal file
@@ -16,10 +16,10 @@ Output formats:
|
|||||||
- prisma: Prisma schema
|
- prisma: Prisma schema
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python -m modelgen from-config -c config.json -o models.py
|
python -m soleprint.station.tools.modelgen from-config -c config.json -o models.py
|
||||||
python -m modelgen from-schema -o models/ --targets pydantic,typescript
|
python -m soleprint.station.tools.modelgen from-schema -o models/ --targets pydantic,typescript
|
||||||
python -m modelgen extract --source /path/to/django --targets pydantic
|
python -m soleprint.station.tools.modelgen extract --source /path/to/django --targets pydantic
|
||||||
python -m modelgen list-formats
|
python -m soleprint.station.tools.modelgen list-formats
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__version__ = "0.2.0"
|
__version__ = "0.2.0"
|
||||||
|
|||||||
@@ -16,10 +16,11 @@ Output formats:
|
|||||||
- prisma: Prisma schema
|
- prisma: Prisma schema
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python -m modelgen --help
|
python -m soleprint.station.tools.modelgen --help
|
||||||
python -m modelgen from-config -c config.json -o models.py
|
python -m soleprint.station.tools.modelgen from-config -c config.json -o models.py
|
||||||
python -m modelgen from-schema -o models/ --targets pydantic,typescript
|
python -m soleprint.station.tools.modelgen from-schema -o models/ --targets pydantic,typescript
|
||||||
python -m modelgen extract --source /path/to/django --targets pydantic
|
python -m soleprint.station.tools.modelgen extract --source /path/to/django --targets pydantic
|
||||||
|
python -m soleprint.station.tools.modelgen generate --config schema/modelgen.json
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
@@ -72,10 +73,24 @@ def cmd_from_schema(args):
|
|||||||
print("that exports DATACLASSES and ENUMS lists.", file=sys.stderr)
|
print("that exports DATACLASSES and ENUMS lists.", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
print(f"Loading schema: {schema_path}")
|
# Parse include groups
|
||||||
schema = load_schema(schema_path)
|
include = None
|
||||||
|
if args.include:
|
||||||
|
include = {g.strip() for g in args.include.split(",")}
|
||||||
|
|
||||||
print(f"Found {len(schema.models)} models, {len(schema.enums)} enums")
|
print(f"Loading schema: {schema_path}")
|
||||||
|
schema = load_schema(schema_path, include=include)
|
||||||
|
|
||||||
|
loaded = []
|
||||||
|
if schema.models:
|
||||||
|
loaded.append(f"{len(schema.models)} models")
|
||||||
|
if schema.enums:
|
||||||
|
loaded.append(f"{len(schema.enums)} enums")
|
||||||
|
if schema.api_models:
|
||||||
|
loaded.append(f"{len(schema.api_models)} api models")
|
||||||
|
if schema.grpc_messages:
|
||||||
|
loaded.append(f"{len(schema.grpc_messages)} grpc messages")
|
||||||
|
print(f"Found {', '.join(loaded)}")
|
||||||
|
|
||||||
# Parse targets
|
# Parse targets
|
||||||
targets = [t.strip() for t in args.targets.split(",")]
|
targets = [t.strip() for t in args.targets.split(",")]
|
||||||
@@ -163,6 +178,47 @@ def cmd_extract(args):
|
|||||||
print("Done!")
|
print("Done!")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_generate(args):
|
||||||
|
"""Generate all targets from a JSON config file."""
|
||||||
|
import json
|
||||||
|
from .loader import load_schema
|
||||||
|
|
||||||
|
config_path = Path(args.config)
|
||||||
|
if not config_path.exists():
|
||||||
|
print(f"Error: Config file not found: {config_path}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
with open(config_path) as f:
|
||||||
|
config = json.load(f)
|
||||||
|
|
||||||
|
# Resolve paths relative to current working directory
|
||||||
|
schema_path = Path(config["schema"])
|
||||||
|
if not schema_path.exists():
|
||||||
|
print(f"Error: Schema folder not found: {schema_path}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(f"Loading schema: {schema_path}")
|
||||||
|
|
||||||
|
for target_conf in config["targets"]:
|
||||||
|
target = target_conf["target"]
|
||||||
|
output = Path(target_conf["output"])
|
||||||
|
include = set(target_conf.get("include", []))
|
||||||
|
name_map = target_conf.get("name_map", {})
|
||||||
|
|
||||||
|
if target not in GENERATORS:
|
||||||
|
print(f"Warning: Unknown target '{target}', skipping", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Load schema with this target's include filter
|
||||||
|
schema = load_schema(schema_path, include=include or None)
|
||||||
|
|
||||||
|
generator = GENERATORS[target](name_map=name_map)
|
||||||
|
print(f"Generating {target} to: {output}")
|
||||||
|
generator.generate(schema, output)
|
||||||
|
|
||||||
|
print("Done!")
|
||||||
|
|
||||||
|
|
||||||
def cmd_list_formats(args):
|
def cmd_list_formats(args):
|
||||||
"""List available output formats."""
|
"""List available output formats."""
|
||||||
print("Available output formats:")
|
print("Available output formats:")
|
||||||
@@ -237,6 +293,12 @@ def main():
|
|||||||
default="pydantic",
|
default="pydantic",
|
||||||
help=f"Comma-separated output targets ({formats_str})",
|
help=f"Comma-separated output targets ({formats_str})",
|
||||||
)
|
)
|
||||||
|
schema_parser.add_argument(
|
||||||
|
"--include",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Comma-separated model groups to include (dataclasses,enums,api,grpc). Default: all.",
|
||||||
|
)
|
||||||
schema_parser.set_defaults(func=cmd_from_schema)
|
schema_parser.set_defaults(func=cmd_from_schema)
|
||||||
|
|
||||||
# extract command
|
# extract command
|
||||||
@@ -275,6 +337,21 @@ def main():
|
|||||||
)
|
)
|
||||||
extract_parser.set_defaults(func=cmd_extract)
|
extract_parser.set_defaults(func=cmd_extract)
|
||||||
|
|
||||||
|
|
||||||
|
# generate command (config-driven multi-target)
|
||||||
|
gen_parser = subparsers.add_parser(
|
||||||
|
"generate",
|
||||||
|
help="Generate all targets from a JSON config file",
|
||||||
|
)
|
||||||
|
gen_parser.add_argument(
|
||||||
|
"--config",
|
||||||
|
"-c",
|
||||||
|
type=str,
|
||||||
|
required=True,
|
||||||
|
help="Path to generation config file (e.g., schema/modelgen.json)",
|
||||||
|
)
|
||||||
|
gen_parser.set_defaults(func=cmd_generate)
|
||||||
|
|
||||||
# list-formats command
|
# list-formats command
|
||||||
formats_parser = subparsers.add_parser(
|
formats_parser = subparsers.add_parser(
|
||||||
"list-formats",
|
"list-formats",
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Supported generators:
|
|||||||
- TypeScriptGenerator: TypeScript interfaces
|
- TypeScriptGenerator: TypeScript interfaces
|
||||||
- ProtobufGenerator: Protocol Buffer definitions
|
- ProtobufGenerator: Protocol Buffer definitions
|
||||||
- PrismaGenerator: Prisma schema
|
- PrismaGenerator: Prisma schema
|
||||||
|
- StrawberryGenerator: Strawberry type/input/enum classes
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Dict, Type
|
from typing import Dict, Type
|
||||||
@@ -16,6 +17,7 @@ from .django import DjangoGenerator
|
|||||||
from .prisma import PrismaGenerator
|
from .prisma import PrismaGenerator
|
||||||
from .protobuf import ProtobufGenerator
|
from .protobuf import ProtobufGenerator
|
||||||
from .pydantic import PydanticGenerator
|
from .pydantic import PydanticGenerator
|
||||||
|
from .strawberry import StrawberryGenerator
|
||||||
from .typescript import TypeScriptGenerator
|
from .typescript import TypeScriptGenerator
|
||||||
|
|
||||||
# Registry of available generators
|
# Registry of available generators
|
||||||
@@ -27,12 +29,14 @@ GENERATORS: Dict[str, Type[BaseGenerator]] = {
|
|||||||
"protobuf": ProtobufGenerator,
|
"protobuf": ProtobufGenerator,
|
||||||
"proto": ProtobufGenerator, # Alias
|
"proto": ProtobufGenerator, # Alias
|
||||||
"prisma": PrismaGenerator,
|
"prisma": PrismaGenerator,
|
||||||
|
"strawberry": StrawberryGenerator,
|
||||||
}
|
}
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"BaseGenerator",
|
"BaseGenerator",
|
||||||
"PydanticGenerator",
|
"PydanticGenerator",
|
||||||
"DjangoGenerator",
|
"DjangoGenerator",
|
||||||
|
"StrawberryGenerator",
|
||||||
"TypeScriptGenerator",
|
"TypeScriptGenerator",
|
||||||
"ProtobufGenerator",
|
"ProtobufGenerator",
|
||||||
"PrismaGenerator",
|
"PrismaGenerator",
|
||||||
|
|||||||
@@ -6,12 +6,19 @@ Abstract base class for all code generators.
|
|||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
|
||||||
class BaseGenerator(ABC):
|
class BaseGenerator(ABC):
|
||||||
"""Abstract base for code generators."""
|
"""Abstract base for code generators."""
|
||||||
|
|
||||||
|
def __init__(self, name_map: Dict[str, str] = None):
|
||||||
|
self.name_map = name_map or {}
|
||||||
|
|
||||||
|
def map_name(self, name: str) -> str:
|
||||||
|
"""Apply name_map to a model name."""
|
||||||
|
return self.name_map.get(name, name)
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def generate(self, models: Any, output_path: Path) -> None:
|
def generate(self, models: Any, output_path: Path) -> None:
|
||||||
"""Generate code for the given models to the specified path."""
|
"""Generate code for the given models to the specified path."""
|
||||||
|
|||||||
@@ -217,12 +217,14 @@ class DjangoGenerator(BaseGenerator):
|
|||||||
|
|
||||||
# Enum
|
# Enum
|
||||||
if isinstance(base, type) and issubclass(base, Enum):
|
if isinstance(base, type) and issubclass(base, Enum):
|
||||||
|
enum_name = base.__name__
|
||||||
extra = []
|
extra = []
|
||||||
if optional:
|
if optional:
|
||||||
extra.append("null=True, blank=True")
|
extra.append("null=True, blank=True")
|
||||||
if default is not dc.MISSING and isinstance(default, Enum):
|
if default is not dc.MISSING and isinstance(default, Enum):
|
||||||
extra.append(f"default=Status.{default.name}")
|
extra.append(f"default={enum_name}.{default.name}")
|
||||||
return DJANGO_TYPES["enum"].format(
|
return DJANGO_TYPES["enum"].format(
|
||||||
|
enum_name=enum_name,
|
||||||
opts=", " + ", ".join(extra) if extra else ""
|
opts=", " + ", ".join(extra) if extra else ""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,12 @@
|
|||||||
Pydantic Generator
|
Pydantic Generator
|
||||||
|
|
||||||
Generates Pydantic BaseModel classes from model definitions.
|
Generates Pydantic BaseModel classes from model definitions.
|
||||||
|
Supports two output modes:
|
||||||
|
- File output: flat models (backwards compatible)
|
||||||
|
- Directory output: CRUD variants (Create/Update/Response) per model
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import dataclasses as dc
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, List, get_type_hints
|
from typing import Any, List, get_type_hints
|
||||||
@@ -13,6 +17,13 @@ from ..loader.schema import EnumDefinition, FieldDefinition, ModelDefinition
|
|||||||
from ..types import PYDANTIC_RESOLVERS
|
from ..types import PYDANTIC_RESOLVERS
|
||||||
from .base import BaseGenerator
|
from .base import BaseGenerator
|
||||||
|
|
||||||
|
# Fields to skip per CRUD variant
|
||||||
|
SKIP_FIELDS = {
|
||||||
|
"Create": {"id", "created_at", "updated_at", "status", "error_message"},
|
||||||
|
"Update": {"id", "created_at", "updated_at"},
|
||||||
|
"Response": set(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class PydanticGenerator(BaseGenerator):
|
class PydanticGenerator(BaseGenerator):
|
||||||
"""Generates Pydantic model files."""
|
"""Generates Pydantic model files."""
|
||||||
@@ -21,52 +32,187 @@ class PydanticGenerator(BaseGenerator):
|
|||||||
return ".py"
|
return ".py"
|
||||||
|
|
||||||
def generate(self, models, output_path: Path) -> None:
|
def generate(self, models, output_path: Path) -> None:
|
||||||
"""Generate Pydantic models to output_path."""
|
"""Generate Pydantic models to output_path.
|
||||||
|
|
||||||
|
If output_path is a directory (or doesn't end in .py), generate
|
||||||
|
multi-file CRUD variants. Otherwise, generate flat models to a
|
||||||
|
single file.
|
||||||
|
"""
|
||||||
|
output_path = Path(output_path)
|
||||||
|
|
||||||
|
if output_path.suffix != ".py":
|
||||||
|
# Directory mode: CRUD variants
|
||||||
|
self._generate_crud_directory(models, output_path)
|
||||||
|
else:
|
||||||
|
# File mode: flat models (backwards compatible)
|
||||||
|
self._generate_flat_file(models, output_path)
|
||||||
|
|
||||||
|
def _generate_flat_file(self, models, output_path: Path) -> None:
|
||||||
|
"""Generate flat models to a single file (original behavior)."""
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# Detect input type and generate accordingly
|
|
||||||
if hasattr(models, "get_shared_component"):
|
if hasattr(models, "get_shared_component"):
|
||||||
# ConfigLoader (soleprint config)
|
|
||||||
content = self._generate_from_config(models)
|
content = self._generate_from_config(models)
|
||||||
elif hasattr(models, "models"):
|
elif hasattr(models, "models"):
|
||||||
# SchemaLoader
|
|
||||||
content = self._generate_from_definitions(
|
content = self._generate_from_definitions(
|
||||||
models.models, getattr(models, "enums", [])
|
models.models, getattr(models, "enums", [])
|
||||||
)
|
)
|
||||||
elif isinstance(models, tuple):
|
elif isinstance(models, tuple):
|
||||||
# (models, enums) tuple from extractor
|
|
||||||
content = self._generate_from_definitions(models[0], models[1])
|
content = self._generate_from_definitions(models[0], models[1])
|
||||||
elif isinstance(models, list):
|
elif isinstance(models, list):
|
||||||
# List of dataclasses (MPR style)
|
|
||||||
content = self._generate_from_dataclasses(models)
|
content = self._generate_from_dataclasses(models)
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unsupported input type: {type(models)}")
|
raise ValueError(f"Unsupported input type: {type(models)}")
|
||||||
|
|
||||||
output_path.write_text(content)
|
output_path.write_text(content)
|
||||||
|
|
||||||
|
def _generate_crud_directory(self, models, output_dir: Path) -> None:
|
||||||
|
"""Generate CRUD variant files in a directory."""
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
if hasattr(models, "models"):
|
||||||
|
model_defs = models.models
|
||||||
|
enum_defs = getattr(models, "enums", [])
|
||||||
|
elif isinstance(models, tuple):
|
||||||
|
model_defs = models[0]
|
||||||
|
enum_defs = models[1]
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported input type for CRUD mode: {type(models)}")
|
||||||
|
|
||||||
|
# base.py
|
||||||
|
base_content = "\n".join([
|
||||||
|
'"""Pydantic Base Schema - GENERATED FILE"""',
|
||||||
|
"",
|
||||||
|
"from pydantic import BaseModel, ConfigDict",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"class BaseSchema(BaseModel):",
|
||||||
|
' """Base schema with ORM mode."""',
|
||||||
|
" model_config = ConfigDict(from_attributes=True)",
|
||||||
|
"",
|
||||||
|
])
|
||||||
|
(output_dir / "base.py").write_text(base_content)
|
||||||
|
|
||||||
|
# Per-model files
|
||||||
|
imports = ["from .base import BaseSchema"]
|
||||||
|
all_exports = ['"BaseSchema"']
|
||||||
|
|
||||||
|
for model_def in model_defs:
|
||||||
|
mapped = self.map_name(model_def.name)
|
||||||
|
module_name = mapped.lower()
|
||||||
|
|
||||||
|
lines = [
|
||||||
|
f'"""{model_def.name} Schemas - GENERATED FILE"""',
|
||||||
|
"",
|
||||||
|
"from datetime import datetime",
|
||||||
|
"from enum import Enum",
|
||||||
|
"from typing import Any, Dict, List, Optional",
|
||||||
|
"from uuid import UUID",
|
||||||
|
"",
|
||||||
|
"from .base import BaseSchema",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Inline enums used by this model
|
||||||
|
model_enums = self._collect_model_enums(model_def, enum_defs)
|
||||||
|
for enum_def in model_enums:
|
||||||
|
lines.append("")
|
||||||
|
lines.extend(self._generate_enum(enum_def))
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# CRUD variants
|
||||||
|
for suffix in ["Create", "Update", "Response"]:
|
||||||
|
lines.append("")
|
||||||
|
lines.extend(self._generate_crud_model(model_def, mapped, suffix))
|
||||||
|
|
||||||
|
lines.append("")
|
||||||
|
content = "\n".join(lines)
|
||||||
|
(output_dir / f"{module_name}.py").write_text(content)
|
||||||
|
|
||||||
|
# Track imports
|
||||||
|
imports.append(
|
||||||
|
f"from .{module_name} import {mapped}Create, {mapped}Update, {mapped}Response"
|
||||||
|
)
|
||||||
|
all_exports.extend([
|
||||||
|
f'"{mapped}Create"', f'"{mapped}Update"', f'"{mapped}Response"'
|
||||||
|
])
|
||||||
|
|
||||||
|
for enum_def in model_enums:
|
||||||
|
imports.append(f"from .{module_name} import {enum_def.name}")
|
||||||
|
all_exports.append(f'"{enum_def.name}"')
|
||||||
|
|
||||||
|
# __init__.py
|
||||||
|
init_content = "\n".join([
|
||||||
|
'"""API Schemas - GENERATED FILE"""',
|
||||||
|
"",
|
||||||
|
*imports,
|
||||||
|
"",
|
||||||
|
f"__all__ = [{', '.join(all_exports)}]",
|
||||||
|
"",
|
||||||
|
])
|
||||||
|
(output_dir / "__init__.py").write_text(init_content)
|
||||||
|
|
||||||
|
def _collect_model_enums(
|
||||||
|
self, model_def: ModelDefinition, enum_defs: List[EnumDefinition]
|
||||||
|
) -> List[EnumDefinition]:
|
||||||
|
"""Find enums referenced by a model's fields."""
|
||||||
|
enum_names = set()
|
||||||
|
for field in model_def.fields:
|
||||||
|
base, _ = unwrap_optional(field.type_hint)
|
||||||
|
if isinstance(base, type) and issubclass(base, Enum):
|
||||||
|
enum_names.add(base.__name__)
|
||||||
|
return [e for e in enum_defs if e.name in enum_names]
|
||||||
|
|
||||||
|
def _generate_crud_model(
|
||||||
|
self, model_def: ModelDefinition, mapped_name: str, suffix: str
|
||||||
|
) -> List[str]:
|
||||||
|
"""Generate a single CRUD variant (Create/Update/Response)."""
|
||||||
|
class_name = f"{mapped_name}{suffix}"
|
||||||
|
skip = SKIP_FIELDS.get(suffix, set())
|
||||||
|
|
||||||
|
lines = [
|
||||||
|
f"class {class_name}(BaseSchema):",
|
||||||
|
f' """{class_name} schema."""',
|
||||||
|
]
|
||||||
|
|
||||||
|
has_fields = False
|
||||||
|
for field in model_def.fields:
|
||||||
|
if field.name.startswith("_") or field.name in skip:
|
||||||
|
continue
|
||||||
|
|
||||||
|
has_fields = True
|
||||||
|
py_type = self._resolve_type(field.type_hint, field.optional)
|
||||||
|
|
||||||
|
# Update variant: all fields optional
|
||||||
|
if suffix == "Update" and "Optional" not in py_type:
|
||||||
|
py_type = f"Optional[{py_type}]"
|
||||||
|
|
||||||
|
default = self._format_default(field.default, "Optional" in py_type)
|
||||||
|
lines.append(f" {field.name}: {py_type}{default}")
|
||||||
|
|
||||||
|
if not has_fields:
|
||||||
|
lines.append(" pass")
|
||||||
|
|
||||||
|
return lines
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# Flat file generation (original behavior)
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
def _generate_from_definitions(
|
def _generate_from_definitions(
|
||||||
self, models: List[ModelDefinition], enums: List[EnumDefinition]
|
self, models: List[ModelDefinition], enums: List[EnumDefinition]
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Generate from ModelDefinition objects (schema/extract mode)."""
|
|
||||||
lines = self._generate_header()
|
lines = self._generate_header()
|
||||||
|
|
||||||
# Generate enums
|
|
||||||
for enum_def in enums:
|
for enum_def in enums:
|
||||||
lines.extend(self._generate_enum(enum_def))
|
lines.extend(self._generate_enum(enum_def))
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# Generate models
|
|
||||||
for model_def in models:
|
for model_def in models:
|
||||||
lines.extend(self._generate_model_from_definition(model_def))
|
lines.extend(self._generate_model_from_definition(model_def))
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
def _generate_from_dataclasses(self, dataclasses: List[type]) -> str:
|
def _generate_from_dataclasses(self, dataclasses: List[type]) -> str:
|
||||||
"""Generate from Python dataclasses (MPR style)."""
|
|
||||||
lines = self._generate_header()
|
lines = self._generate_header()
|
||||||
|
|
||||||
# Collect and generate enums first
|
|
||||||
enums_generated = set()
|
enums_generated = set()
|
||||||
for cls in dataclasses:
|
for cls in dataclasses:
|
||||||
hints = get_type_hints(cls)
|
hints = get_type_hints(cls)
|
||||||
@@ -77,16 +223,12 @@ class PydanticGenerator(BaseGenerator):
|
|||||||
lines.extend(self._generate_enum_from_python(base))
|
lines.extend(self._generate_enum_from_python(base))
|
||||||
lines.append("")
|
lines.append("")
|
||||||
enums_generated.add(base.__name__)
|
enums_generated.add(base.__name__)
|
||||||
|
|
||||||
# Generate models
|
|
||||||
for cls in dataclasses:
|
for cls in dataclasses:
|
||||||
lines.extend(self._generate_model_from_dataclass(cls))
|
lines.extend(self._generate_model_from_dataclass(cls))
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
def _generate_header(self) -> List[str]:
|
def _generate_header(self) -> List[str]:
|
||||||
"""Generate file header."""
|
|
||||||
return [
|
return [
|
||||||
'"""',
|
'"""',
|
||||||
"Pydantic Models - GENERATED FILE",
|
"Pydantic Models - GENERATED FILE",
|
||||||
@@ -104,27 +246,23 @@ class PydanticGenerator(BaseGenerator):
|
|||||||
]
|
]
|
||||||
|
|
||||||
def _generate_enum(self, enum_def: EnumDefinition) -> List[str]:
|
def _generate_enum(self, enum_def: EnumDefinition) -> List[str]:
|
||||||
"""Generate Pydantic enum from EnumDefinition."""
|
|
||||||
lines = [f"class {enum_def.name}(str, Enum):"]
|
lines = [f"class {enum_def.name}(str, Enum):"]
|
||||||
for name, value in enum_def.values:
|
for name, value in enum_def.values:
|
||||||
lines.append(f' {name} = "{value}"')
|
lines.append(f' {name} = "{value}"')
|
||||||
return lines
|
return lines
|
||||||
|
|
||||||
def _generate_enum_from_python(self, enum_cls: type) -> List[str]:
|
def _generate_enum_from_python(self, enum_cls: type) -> List[str]:
|
||||||
"""Generate Pydantic enum from Python Enum."""
|
|
||||||
lines = [f"class {enum_cls.__name__}(str, Enum):"]
|
lines = [f"class {enum_cls.__name__}(str, Enum):"]
|
||||||
for member in enum_cls:
|
for member in enum_cls:
|
||||||
lines.append(f' {member.name} = "{member.value}"')
|
lines.append(f' {member.name} = "{member.value}"')
|
||||||
return lines
|
return lines
|
||||||
|
|
||||||
def _generate_model_from_definition(self, model_def: ModelDefinition) -> List[str]:
|
def _generate_model_from_definition(self, model_def: ModelDefinition) -> List[str]:
|
||||||
"""Generate Pydantic model from ModelDefinition."""
|
|
||||||
docstring = model_def.docstring or model_def.name
|
docstring = model_def.docstring or model_def.name
|
||||||
lines = [
|
lines = [
|
||||||
f"class {model_def.name}(BaseModel):",
|
f"class {model_def.name}(BaseModel):",
|
||||||
f' """{docstring.strip().split(chr(10))[0]}"""',
|
f' """{docstring.strip().split(chr(10))[0]}"""',
|
||||||
]
|
]
|
||||||
|
|
||||||
if not model_def.fields:
|
if not model_def.fields:
|
||||||
lines.append(" pass")
|
lines.append(" pass")
|
||||||
else:
|
else:
|
||||||
@@ -132,46 +270,34 @@ class PydanticGenerator(BaseGenerator):
|
|||||||
py_type = self._resolve_type(field.type_hint, field.optional)
|
py_type = self._resolve_type(field.type_hint, field.optional)
|
||||||
default = self._format_default(field.default, field.optional)
|
default = self._format_default(field.default, field.optional)
|
||||||
lines.append(f" {field.name}: {py_type}{default}")
|
lines.append(f" {field.name}: {py_type}{default}")
|
||||||
|
|
||||||
return lines
|
return lines
|
||||||
|
|
||||||
def _generate_model_from_dataclass(self, cls: type) -> List[str]:
|
def _generate_model_from_dataclass(self, cls: type) -> List[str]:
|
||||||
"""Generate Pydantic model from a dataclass."""
|
|
||||||
import dataclasses as dc
|
|
||||||
|
|
||||||
docstring = cls.__doc__ or cls.__name__
|
docstring = cls.__doc__ or cls.__name__
|
||||||
lines = [
|
lines = [
|
||||||
f"class {cls.__name__}(BaseModel):",
|
f"class {cls.__name__}(BaseModel):",
|
||||||
f' """{docstring.strip().split(chr(10))[0]}"""',
|
f' """{docstring.strip().split(chr(10))[0]}"""',
|
||||||
]
|
]
|
||||||
|
|
||||||
hints = get_type_hints(cls)
|
hints = get_type_hints(cls)
|
||||||
fields = {f.name: f for f in dc.fields(cls)}
|
fields = {f.name: f for f in dc.fields(cls)}
|
||||||
|
|
||||||
for name, type_hint in hints.items():
|
for name, type_hint in hints.items():
|
||||||
if name.startswith("_"):
|
if name.startswith("_"):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
field = fields.get(name)
|
field = fields.get(name)
|
||||||
default_val = dc.MISSING
|
default_val = dc.MISSING
|
||||||
if field:
|
if field:
|
||||||
if field.default is not dc.MISSING:
|
if field.default is not dc.MISSING:
|
||||||
default_val = field.default
|
default_val = field.default
|
||||||
|
|
||||||
py_type = self._resolve_type(type_hint, False)
|
py_type = self._resolve_type(type_hint, False)
|
||||||
default = self._format_default(default_val, "Optional" in py_type)
|
default = self._format_default(default_val, "Optional" in py_type)
|
||||||
lines.append(f" {name}: {py_type}{default}")
|
lines.append(f" {name}: {py_type}{default}")
|
||||||
|
|
||||||
return lines
|
return lines
|
||||||
|
|
||||||
def _resolve_type(self, type_hint: Any, optional: bool) -> str:
|
def _resolve_type(self, type_hint: Any, optional: bool) -> str:
|
||||||
"""Resolve Python type to Pydantic type string."""
|
|
||||||
base, is_optional = unwrap_optional(type_hint)
|
base, is_optional = unwrap_optional(type_hint)
|
||||||
optional = optional or is_optional
|
optional = optional or is_optional
|
||||||
origin = get_origin_name(base)
|
origin = get_origin_name(base)
|
||||||
type_name = get_type_name(base)
|
type_name = get_type_name(base)
|
||||||
|
|
||||||
# Look up resolver
|
|
||||||
resolver = (
|
resolver = (
|
||||||
PYDANTIC_RESOLVERS.get(origin)
|
PYDANTIC_RESOLVERS.get(origin)
|
||||||
or PYDANTIC_RESOLVERS.get(type_name)
|
or PYDANTIC_RESOLVERS.get(type_name)
|
||||||
@@ -182,14 +308,10 @@ class PydanticGenerator(BaseGenerator):
|
|||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
result = resolver(base) if resolver else "str"
|
result = resolver(base) if resolver else "str"
|
||||||
return f"Optional[{result}]" if optional else result
|
return f"Optional[{result}]" if optional else result
|
||||||
|
|
||||||
def _format_default(self, default: Any, optional: bool) -> str:
|
def _format_default(self, default: Any, optional: bool) -> str:
|
||||||
"""Format default value for field."""
|
|
||||||
import dataclasses as dc
|
|
||||||
|
|
||||||
if optional:
|
if optional:
|
||||||
return " = None"
|
return " = None"
|
||||||
if default is dc.MISSING or default is None:
|
if default is dc.MISSING or default is None:
|
||||||
@@ -204,7 +326,6 @@ class PydanticGenerator(BaseGenerator):
|
|||||||
|
|
||||||
def _generate_from_config(self, config) -> str:
|
def _generate_from_config(self, config) -> str:
|
||||||
"""Generate from ConfigLoader (soleprint config.json mode)."""
|
"""Generate from ConfigLoader (soleprint config.json mode)."""
|
||||||
# Get component names from config
|
|
||||||
config_comp = config.get_shared_component("config")
|
config_comp = config.get_shared_component("config")
|
||||||
data_comp = config.get_shared_component("data")
|
data_comp = config.get_shared_component("data")
|
||||||
|
|
||||||
|
|||||||
220
modelgen/generator/strawberry.py
Normal file
@@ -0,0 +1,220 @@
|
|||||||
|
"""
|
||||||
|
Strawberry Generator
|
||||||
|
|
||||||
|
Generates strawberry type, input, and enum classes from model definitions.
|
||||||
|
Only generates type definitions — queries, mutations, and resolvers are hand-written.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import dataclasses as dc
|
||||||
|
from enum import Enum
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, List, get_type_hints
|
||||||
|
|
||||||
|
from ..helpers import get_origin_name, get_type_name, unwrap_optional
|
||||||
|
from ..loader.schema import EnumDefinition, FieldDefinition, ModelDefinition
|
||||||
|
from ..types import STRAWBERRY_RESOLVERS
|
||||||
|
from .base import BaseGenerator
|
||||||
|
|
||||||
|
|
||||||
|
class StrawberryGenerator(BaseGenerator):
|
||||||
|
"""Generates strawberry type definition files."""
|
||||||
|
|
||||||
|
def file_extension(self) -> str:
|
||||||
|
return ".py"
|
||||||
|
|
||||||
|
def generate(self, models, output_path: Path) -> None:
|
||||||
|
"""Generate strawberry types to output_path."""
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
if hasattr(models, "models"):
|
||||||
|
# SchemaLoader
|
||||||
|
content = self._generate_from_definitions(
|
||||||
|
models.models,
|
||||||
|
getattr(models, "enums", []),
|
||||||
|
getattr(models, "api_models", []),
|
||||||
|
)
|
||||||
|
elif isinstance(models, tuple):
|
||||||
|
content = self._generate_from_definitions(models[0], models[1], [])
|
||||||
|
elif isinstance(models, list):
|
||||||
|
content = self._generate_from_dataclasses(models)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported input type: {type(models)}")
|
||||||
|
|
||||||
|
output_path.write_text(content)
|
||||||
|
|
||||||
|
def _generate_from_definitions(
|
||||||
|
self,
|
||||||
|
models: List[ModelDefinition],
|
||||||
|
enums: List[EnumDefinition],
|
||||||
|
api_models: List[ModelDefinition],
|
||||||
|
) -> str:
|
||||||
|
lines = self._generate_header()
|
||||||
|
|
||||||
|
for enum_def in enums:
|
||||||
|
lines.extend(self._generate_enum(enum_def))
|
||||||
|
lines.append("")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
for model_def in models:
|
||||||
|
lines.extend(self._generate_object_type(model_def))
|
||||||
|
lines.append("")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
for model_def in api_models:
|
||||||
|
if model_def.name.endswith("Request"):
|
||||||
|
lines.extend(self._generate_input_type(model_def))
|
||||||
|
else:
|
||||||
|
lines.extend(self._generate_object_type(model_def))
|
||||||
|
lines.append("")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
return "\n".join(lines).rstrip() + "\n"
|
||||||
|
|
||||||
|
def _generate_from_dataclasses(self, dataclasses: List[type]) -> str:
|
||||||
|
lines = self._generate_header()
|
||||||
|
|
||||||
|
enums_generated = set()
|
||||||
|
for cls in dataclasses:
|
||||||
|
hints = get_type_hints(cls)
|
||||||
|
for type_hint in hints.values():
|
||||||
|
base, _ = unwrap_optional(type_hint)
|
||||||
|
if isinstance(base, type) and issubclass(base, Enum):
|
||||||
|
if base.__name__ not in enums_generated:
|
||||||
|
lines.extend(self._generate_enum_from_python(base))
|
||||||
|
lines.append("")
|
||||||
|
lines.append("")
|
||||||
|
enums_generated.add(base.__name__)
|
||||||
|
|
||||||
|
for cls in dataclasses:
|
||||||
|
lines.extend(self._generate_object_type_from_dataclass(cls))
|
||||||
|
lines.append("")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
return "\n".join(lines).rstrip() + "\n"
|
||||||
|
|
||||||
|
def _generate_header(self) -> List[str]:
|
||||||
|
return [
|
||||||
|
'"""',
|
||||||
|
"Strawberry Types - GENERATED FILE",
|
||||||
|
"",
|
||||||
|
"Do not edit directly. Regenerate using modelgen.",
|
||||||
|
'"""',
|
||||||
|
"",
|
||||||
|
"import strawberry",
|
||||||
|
"from enum import Enum",
|
||||||
|
"from typing import List, Optional",
|
||||||
|
"from uuid import UUID",
|
||||||
|
"from datetime import datetime",
|
||||||
|
"from strawberry.scalars import JSON",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
def _generate_enum(self, enum_def: EnumDefinition) -> List[str]:
|
||||||
|
lines = ["@strawberry.enum", f"class {enum_def.name}(Enum):"]
|
||||||
|
for name, value in enum_def.values:
|
||||||
|
lines.append(f' {name} = "{value}"')
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def _generate_enum_from_python(self, enum_cls: type) -> List[str]:
|
||||||
|
lines = ["@strawberry.enum", f"class {enum_cls.__name__}(Enum):"]
|
||||||
|
for member in enum_cls:
|
||||||
|
lines.append(f' {member.name} = "{member.value}"')
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def _generate_object_type(self, model_def: ModelDefinition) -> List[str]:
|
||||||
|
name = model_def.name
|
||||||
|
type_name = f"{name}Type" if not name.endswith("Type") else name
|
||||||
|
|
||||||
|
lines = ["@strawberry.type", f"class {type_name}:"]
|
||||||
|
if model_def.docstring:
|
||||||
|
doc = model_def.docstring.strip().split("\n")[0]
|
||||||
|
lines.append(f' """{doc}"""')
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
if not model_def.fields:
|
||||||
|
lines.append(" pass")
|
||||||
|
else:
|
||||||
|
for field in model_def.fields:
|
||||||
|
type_str = self._resolve_type(field.type_hint, optional=True)
|
||||||
|
lines.append(f" {field.name}: {type_str} = None")
|
||||||
|
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def _generate_input_type(self, model_def: ModelDefinition) -> List[str]:
|
||||||
|
name = model_def.name
|
||||||
|
if name.endswith("Request"):
|
||||||
|
input_name = name[: -len("Request")] + "Input"
|
||||||
|
else:
|
||||||
|
input_name = f"{name}Input"
|
||||||
|
|
||||||
|
lines = ["@strawberry.input", f"class {input_name}:"]
|
||||||
|
if model_def.docstring:
|
||||||
|
doc = model_def.docstring.strip().split("\n")[0]
|
||||||
|
lines.append(f' """{doc}"""')
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
if not model_def.fields:
|
||||||
|
lines.append(" pass")
|
||||||
|
else:
|
||||||
|
# Required fields first, then optional/defaulted
|
||||||
|
required = []
|
||||||
|
optional = []
|
||||||
|
for field in model_def.fields:
|
||||||
|
has_default = field.default is not dc.MISSING
|
||||||
|
if not field.optional and not has_default:
|
||||||
|
required.append(field)
|
||||||
|
else:
|
||||||
|
optional.append(field)
|
||||||
|
|
||||||
|
for field in required:
|
||||||
|
type_str = self._resolve_type(field.type_hint, optional=False)
|
||||||
|
lines.append(f" {field.name}: {type_str}")
|
||||||
|
|
||||||
|
for field in optional:
|
||||||
|
has_default = field.default is not dc.MISSING
|
||||||
|
if has_default and not callable(field.default):
|
||||||
|
type_str = self._resolve_type(field.type_hint, optional=False)
|
||||||
|
lines.append(f" {field.name}: {type_str} = {field.default!r}")
|
||||||
|
else:
|
||||||
|
type_str = self._resolve_type(field.type_hint, optional=True)
|
||||||
|
lines.append(f" {field.name}: {type_str} = None")
|
||||||
|
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def _generate_object_type_from_dataclass(self, cls: type) -> List[str]:
|
||||||
|
type_name = f"{cls.__name__}Type"
|
||||||
|
lines = ["@strawberry.type", f"class {type_name}:"]
|
||||||
|
|
||||||
|
hints = get_type_hints(cls)
|
||||||
|
for name, type_hint in hints.items():
|
||||||
|
if name.startswith("_"):
|
||||||
|
continue
|
||||||
|
type_str = self._resolve_type(type_hint, optional=True)
|
||||||
|
lines.append(f" {name}: {type_str} = None")
|
||||||
|
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def _resolve_type(self, type_hint: Any, optional: bool) -> str:
|
||||||
|
"""Resolve Python type hint to a strawberry annotation string."""
|
||||||
|
base, is_optional = unwrap_optional(type_hint)
|
||||||
|
optional = optional or is_optional
|
||||||
|
origin = get_origin_name(base)
|
||||||
|
type_name = get_type_name(base)
|
||||||
|
|
||||||
|
resolver = (
|
||||||
|
STRAWBERRY_RESOLVERS.get(origin)
|
||||||
|
or STRAWBERRY_RESOLVERS.get(type_name)
|
||||||
|
or STRAWBERRY_RESOLVERS.get(base)
|
||||||
|
or (
|
||||||
|
STRAWBERRY_RESOLVERS["enum"]
|
||||||
|
if isinstance(base, type) and issubclass(base, Enum)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
inner = resolver(base) if resolver else "str"
|
||||||
|
|
||||||
|
if optional:
|
||||||
|
return f"Optional[{inner}]"
|
||||||
|
return inner
|
||||||