From dee8bbfb10d1e12ac56b30012a8268757b68c07c Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Sun, 1 Feb 2026 12:39:21 -0300 Subject: [PATCH] architecture docs --- docs/architecture/01-system-overview.dot | 105 ++++++++ docs/architecture/01-system-overview.svg | 260 ++++++++++++++++++++ docs/architecture/02-data-model.dot | 22 ++ docs/architecture/02-data-model.svg | 127 ++++++++++ docs/architecture/03-job-flow.dot | 101 ++++++++ docs/architecture/03-job-flow.svg | 296 +++++++++++++++++++++++ docs/architecture/index.html | 101 ++++++++ docs/architecture/styles.css | 143 +++++++++++ 8 files changed, 1155 insertions(+) create mode 100644 docs/architecture/01-system-overview.dot create mode 100644 docs/architecture/01-system-overview.svg create mode 100644 docs/architecture/02-data-model.dot create mode 100644 docs/architecture/02-data-model.svg create mode 100644 docs/architecture/03-job-flow.dot create mode 100644 docs/architecture/03-job-flow.svg create mode 100644 docs/architecture/index.html create mode 100644 docs/architecture/styles.css diff --git a/docs/architecture/01-system-overview.dot b/docs/architecture/01-system-overview.dot new file mode 100644 index 0000000..2ee63e0 --- /dev/null +++ b/docs/architecture/01-system-overview.dot @@ -0,0 +1,105 @@ +digraph system_overview { + rankdir=TB + node [shape=box, style=rounded, fontname="Helvetica"] + edge [fontname="Helvetica", fontsize=10] + + // Title + labelloc="t" + label="MPR - System Overview" + fontsize=16 + fontname="Helvetica-Bold" + + // Styling + graph [splines=ortho, nodesep=0.8, ranksep=0.8] + + // External + subgraph cluster_external { + label="External" + style=dashed + color=gray + + browser [label="Browser\nmpr.local.ar", shape=ellipse] + } + + // Nginx reverse proxy + subgraph cluster_proxy { + label="Reverse Proxy" + style=filled + fillcolor="#e8f4f8" + + nginx [label="nginx\nport 80"] + } + + // Application layer + subgraph cluster_apps { + label="Application Layer" + style=filled + fillcolor="#f0f8e8" + + django [label="Django\n/admin\nport 8701"] + fastapi [label="FastAPI\n/api\nport 8702"] + timeline [label="Timeline UI\n/ui\nport 5173"] + } + + // Worker layer + subgraph cluster_workers { + label="Worker Layer" + style=filled + fillcolor="#fff8e8" + + grpc_server [label="gRPC Server\nport 50051"] + celery [label="Celery Worker\n(local)"] + lambda [label="Lambda\n(cloud)", style="dashed,rounded"] + } + + // Data layer + subgraph cluster_data { + label="Data Layer" + style=filled + fillcolor="#f8e8f0" + + postgres [label="PostgreSQL\nport 5433", shape=cylinder] + redis [label="Redis\nport 6380", shape=cylinder] + sqs [label="SQS\n(cloud)", shape=cylinder, style=dashed] + } + + // Storage + subgraph cluster_storage { + label="File Storage" + style=filled + fillcolor="#f0f0f0" + + local_fs [label="Local FS\n/media", shape=folder] + s3 [label="S3\n(cloud)", shape=folder, style=dashed] + } + + // Connections + browser -> nginx + + nginx -> django [label="/admin"] + nginx -> fastapi [label="/api"] + nginx -> timeline [label="/ui"] + + // Django uses FastAPI for operations (single API gateway) + django -> fastapi [label="job operations"] + django -> postgres [label="CRUD only"] + + // Timeline UI uses FastAPI + timeline -> fastapi [label="REST API"] + + // FastAPI is the single API gateway + fastapi -> postgres + fastapi -> redis [label="job status"] + fastapi -> grpc_server [label="gRPC\nprogress streaming"] + + // Worker layer + grpc_server -> celery [label="task dispatch"] + celery -> redis [label="queue"] + celery -> postgres [label="job updates"] + celery -> grpc_server [label="progress\ncallbacks", style=dotted] + celery -> local_fs [label="read/write"] + + // Cloud (future) + lambda -> sqs [label="queue", style=dashed] + lambda -> s3 [label="read/write", style=dashed] +} diff --git a/docs/architecture/01-system-overview.svg b/docs/architecture/01-system-overview.svg new file mode 100644 index 0000000..abe50ac --- /dev/null +++ b/docs/architecture/01-system-overview.svg @@ -0,0 +1,260 @@ + + + + + + +system_overview + +MPR - System Overview + +cluster_external + +External + + +cluster_proxy + +Reverse Proxy + + +cluster_apps + +Application Layer + + +cluster_workers + +Worker Layer + + +cluster_data + +Data Layer + + +cluster_storage + +File Storage + + + +browser + +Browser +mpr.local.ar + + + +nginx + +nginx +port 80 + + + +browser->nginx + + + + + +django + +Django +/admin +port 8701 + + + +nginx->django + + +/admin + + + +fastapi + +FastAPI +/api +port 8702 + + + +nginx->fastapi + + +/api + + + +timeline + +Timeline UI +/ui +port 5173 + + + +nginx->timeline + + +/ui + + + +django->fastapi + + +job operations + + + +postgres + + +PostgreSQL +port 5433 + + + +django->postgres + + +CRUD only + + + +grpc_server + +gRPC Server +port 50051 + + + +fastapi->grpc_server + + +gRPC +progress streaming + + + +fastapi->postgres + + + + + +redis + + +Redis +port 6380 + + + +fastapi->redis + + +job status + + + +timeline->fastapi + + +REST API + + + +celery + +Celery Worker +(local) + + + +grpc_server->celery + + +task dispatch + + + +celery->grpc_server + + +progress +callbacks + + + +celery->postgres + + +job updates + + + +celery->redis + + +queue + + + +local_fs + +Local FS +/media + + + +celery->local_fs + + +read/write + + + +lambda + +Lambda +(cloud) + + + +sqs + + +SQS +(cloud) + + + +lambda->sqs + + +queue + + + +s3 + +S3 +(cloud) + + + +lambda->s3 + + +read/write + + + diff --git a/docs/architecture/02-data-model.dot b/docs/architecture/02-data-model.dot new file mode 100644 index 0000000..b0348d1 --- /dev/null +++ b/docs/architecture/02-data-model.dot @@ -0,0 +1,22 @@ +digraph data_model { + rankdir=LR + node [shape=record, fontname="Helvetica", fontsize=11] + edge [fontname="Helvetica", fontsize=10] + + labelloc="t" + label="MPR - Data Model" + fontsize=16 + fontname="Helvetica-Bold" + + graph [splines=ortho, nodesep=0.6, ranksep=1.2] + + MediaAsset [label="{MediaAsset|id: UUID (PK)\lfilename: str\lfile_path: str\lfile_size: int?\lstatus: pending/ready/error\lerror_message: str?\l|duration: float?\lvideo_codec: str?\laudio_codec: str?\lwidth: int?\lheight: int?\lframerate: float?\lbitrate: int?\lproperties: JSON\l|comments: str\ltags: JSON[]\l|created_at: datetime\lupdated_at: datetime\l}"] + + TranscodePreset [label="{TranscodePreset|id: UUID (PK)\lname: str (unique)\ldescription: str\lis_builtin: bool\l|container: str\l|video_codec: str\lvideo_bitrate: str?\lvideo_crf: int?\lvideo_preset: str?\lresolution: str?\lframerate: float?\l|audio_codec: str\laudio_bitrate: str?\laudio_channels: int?\laudio_samplerate: int?\l|extra_args: JSON[]\l|created_at: datetime\lupdated_at: datetime\l}"] + + TranscodeJob [label="{TranscodeJob|id: UUID (PK)\l|source_asset_id: UUID (FK)\l|preset_id: UUID? (FK)\lpreset_snapshot: JSON\l|trim_start: float?\ltrim_end: float?\l|output_filename: str\loutput_path: str?\loutput_asset_id: UUID? (FK)\l|status: pending/processing/...\lprogress: float (0-100)\lcurrent_frame: int?\lcurrent_time: float?\lspeed: str?\lerror_message: str?\l|celery_task_id: str?\lpriority: int\l|created_at: datetime\lstarted_at: datetime?\lcompleted_at: datetime?\l}"] + + MediaAsset -> TranscodeJob [label="1:N source_asset"] + TranscodePreset -> TranscodeJob [label="1:N preset"] + TranscodeJob -> MediaAsset [label="1:1 output_asset", style=dashed] +} diff --git a/docs/architecture/02-data-model.svg b/docs/architecture/02-data-model.svg new file mode 100644 index 0000000..4fdd5ed --- /dev/null +++ b/docs/architecture/02-data-model.svg @@ -0,0 +1,127 @@ + + + + + + +data_model + +MPR - Data Model + + +MediaAsset + +MediaAsset + +id: UUID (PK) +filename: str +file_path: str +file_size: int? +status: pending/ready/error +error_message: str? + +duration: float? +video_codec: str? +audio_codec: str? +width: int? +height: int? +framerate: float? +bitrate: int? +properties: JSON + +comments: str +tags: JSON[] + +created_at: datetime +updated_at: datetime + + + +TranscodeJob + +TranscodeJob + +id: UUID (PK) + +source_asset_id: UUID (FK) + +preset_id: UUID? (FK) +preset_snapshot: JSON + +trim_start: float? +trim_end: float? + +output_filename: str +output_path: str? +output_asset_id: UUID? (FK) + +status: pending/processing/... +progress: float (0-100) +current_frame: int? +current_time: float? +speed: str? +error_message: str? + +celery_task_id: str? +priority: int + +created_at: datetime +started_at: datetime? +completed_at: datetime? + + + +MediaAsset->TranscodeJob + + +1:N source_asset + + + +TranscodePreset + +TranscodePreset + +id: UUID (PK) +name: str (unique) +description: str +is_builtin: bool + +container: str + +video_codec: str +video_bitrate: str? +video_crf: int? +video_preset: str? +resolution: str? +framerate: float? + +audio_codec: str +audio_bitrate: str? +audio_channels: int? +audio_samplerate: int? + +extra_args: JSON[] + +created_at: datetime +updated_at: datetime + + + +TranscodePreset->TranscodeJob + + +1:N preset + + + +TranscodeJob->MediaAsset + + +1:1 output_asset + + + diff --git a/docs/architecture/03-job-flow.dot b/docs/architecture/03-job-flow.dot new file mode 100644 index 0000000..813da0c --- /dev/null +++ b/docs/architecture/03-job-flow.dot @@ -0,0 +1,101 @@ +digraph job_flow { + rankdir=TB + node [shape=box, style=rounded, fontname="Helvetica"] + edge [fontname="Helvetica", fontsize=10] + + // Title + labelloc="t" + label="MPR - Job Flow" + fontsize=16 + fontname="Helvetica-Bold" + + graph [splines=ortho, nodesep=0.6, ranksep=0.6] + + // States + subgraph cluster_states { + label="Job States" + style=filled + fillcolor="#f8f8f8" + + pending [label="PENDING", fillcolor="#ffc107", style="filled,rounded"] + processing [label="PROCESSING", fillcolor="#17a2b8", style="filled,rounded", fontcolor=white] + completed [label="COMPLETED", fillcolor="#28a745", style="filled,rounded", fontcolor=white] + failed [label="FAILED", fillcolor="#dc3545", style="filled,rounded", fontcolor=white] + cancelled [label="CANCELLED", fillcolor="#6c757d", style="filled,rounded", fontcolor=white] + } + + // Transitions + pending -> processing [label="worker picks up"] + processing -> completed [label="success"] + processing -> failed [label="error"] + pending -> cancelled [label="user cancels"] + processing -> cancelled [label="user cancels"] + failed -> pending [label="retry"] + + // API actions + subgraph cluster_api { + label="API Actions" + style=dashed + color=gray + + create_job [label="POST /jobs/", shape=ellipse] + cancel_job [label="POST /jobs/{id}/cancel", shape=ellipse] + retry_job [label="POST /jobs/{id}/retry", shape=ellipse] + } + + create_job -> pending + cancel_job -> cancelled [style=dashed] + retry_job -> pending [style=dashed] + + // Executor layer + subgraph cluster_executor { + label="Executor Layer" + style=filled + fillcolor="#fff8e8" + + executor [label="Executor\n(abstract)", shape=diamond] + local [label="LocalExecutor\nCelery + FFmpeg"] + lambda_exec [label="LambdaExecutor\nSQS + Lambda"] + } + + processing -> executor + executor -> local [label="MPR_EXECUTOR=local"] + executor -> lambda_exec [label="MPR_EXECUTOR=lambda", style=dashed] + + // FFmpeg operations + subgraph cluster_ffmpeg { + label="FFmpeg Operations" + style=filled + fillcolor="#e8f4e8" + + transcode [label="Transcode\n(with preset)"] + trim [label="Trim\n(-c:v copy -c:a copy)"] + } + + local -> transcode + local -> trim + + // gRPC streaming + subgraph cluster_grpc { + label="gRPC Communication" + style=filled + fillcolor="#e8e8f8" + + grpc_stream [label="StreamProgress\n(server streaming)", shape=parallelogram] + grpc_submit [label="SubmitJob\n(unary)", shape=parallelogram] + grpc_cancel [label="CancelJob\n(unary)", shape=parallelogram] + } + + // Progress tracking via gRPC + progress [label="Progress Updates\n(gRPC → Redis → DB)", shape=note] + transcode -> progress [style=dotted] + trim -> progress [style=dotted] + progress -> grpc_stream [style=dotted, label="stream to client"] + grpc_stream -> processing [style=dotted, label="update status"] + + // gRPC job control + create_job -> grpc_submit [label="via gRPC"] + grpc_submit -> pending [style=dashed] + cancel_job -> grpc_cancel [label="via gRPC"] + grpc_cancel -> cancelled [style=dashed] +} diff --git a/docs/architecture/03-job-flow.svg b/docs/architecture/03-job-flow.svg new file mode 100644 index 0000000..36a21bc --- /dev/null +++ b/docs/architecture/03-job-flow.svg @@ -0,0 +1,296 @@ + + + + + + +job_flow + +MPR - Job Flow + +cluster_states + +Job States + + +cluster_api + +API Actions + + +cluster_executor + +Executor Layer + + +cluster_ffmpeg + +FFmpeg Operations + + +cluster_grpc + +gRPC Communication + + + +pending + +PENDING + + + +processing + +PROCESSING + + + +pending->processing + + +worker picks up + + + +cancelled + +CANCELLED + + + +pending->cancelled + + +user cancels + + + +completed + +COMPLETED + + + +processing->completed + + +success + + + +failed + +FAILED + + + +processing->failed + + +error + + + +processing->cancelled + + +user cancels + + + +executor + +Executor +(abstract) + + + +processing->executor + + + + + +failed->pending + + +retry + + + +create_job + +POST /jobs/ + + + +create_job->pending + + + + + +grpc_submit + +SubmitJob +(unary) + + + +create_job->grpc_submit + + +via gRPC + + + +cancel_job + +POST /jobs/{id}/cancel + + + +cancel_job->cancelled + + + + + +grpc_cancel + +CancelJob +(unary) + + + +cancel_job->grpc_cancel + + +via gRPC + + + +retry_job + +POST /jobs/{id}/retry + + + +retry_job->pending + + + + + +local + +LocalExecutor +Celery + FFmpeg + + + +executor->local + + +MPR_EXECUTOR=local + + + +lambda_exec + +LambdaExecutor +SQS + Lambda + + + +executor->lambda_exec + + +MPR_EXECUTOR=lambda + + + +transcode + +Transcode +(with preset) + + + +local->transcode + + + + + +trim + +Trim +(-c:v copy -c:a copy) + + + +local->trim + + + + + +progress + + + +Progress Updates +(gRPC → Redis → DB) + + + +transcode->progress + + + + + +trim->progress + + + + + +grpc_stream + +StreamProgress +(server streaming) + + + +grpc_stream->processing + + +update status + + + +grpc_submit->pending + + + + + +grpc_cancel->cancelled + + + + + +progress->grpc_stream + + +stream to client + + + diff --git a/docs/architecture/index.html b/docs/architecture/index.html new file mode 100644 index 0000000..5ed7e13 --- /dev/null +++ b/docs/architecture/index.html @@ -0,0 +1,101 @@ + + + + + + MPR - Architecture + + + +

MPR - Media Processor

+

A web-based media transcoding tool with professional architecture.

+ + + +

System Overview

+
+
+

Architecture

+ + System Overview + + Open full size +
+
+ +
+

Components

+ +
+ +

Data Model

+
+
+

Entity Relationships

+ + Data Model + + Open full size +
+
+ +
+

Entities

+ +
+ +

Job Flow

+
+
+

Job Lifecycle

+ + Job Flow + + Open full size +
+
+ +
+

Job States

+ +
+ +

Quick Reference

+ 
# Generate SVGs from DOT files
+dot -Tsvg 01-system-overview.dot -o 01-system-overview.svg
+dot -Tsvg 02-data-model.dot -o 02-data-model.svg
+dot -Tsvg 03-job-flow.dot -o 03-job-flow.svg
+
+# Or generate all at once
+for f in *.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done
+ +

Access Points

+ 
# Add to /etc/hosts
+127.0.0.1 mpr.local.ar
+
+# URLs
+http://mpr.local.ar/admin  - Django Admin
+http://mpr.local.ar/api    - FastAPI (docs at /api/docs)
+http://mpr.local.ar/ui     - Timeline UI
+ + diff --git a/docs/architecture/styles.css b/docs/architecture/styles.css new file mode 100644 index 0000000..ef23579 --- /dev/null +++ b/docs/architecture/styles.css @@ -0,0 +1,143 @@ +:root { + --bg-color: #1a1a2e; + --text-color: #e8e8e8; + --accent-color: #4a90d9; + --border-color: #333; +} + +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif; + background-color: var(--bg-color); + color: var(--text-color); + line-height: 1.6; + padding: 2rem; +} + +h1 { + font-size: 2rem; + margin-bottom: 1rem; + color: var(--accent-color); +} + +h2 { + font-size: 1.5rem; + margin: 2rem 0 1rem; + color: var(--text-color); + border-bottom: 1px solid var(--border-color); + padding-bottom: 0.5rem; +} + +.diagram-container { + display: flex; + flex-wrap: wrap; + gap: 2rem; + margin-top: 1rem; +} + +.diagram { + flex: 1; + min-width: 400px; + background: #252540; + border-radius: 8px; + padding: 1rem; + border: 1px solid var(--border-color); +} + +.diagram h3 { + font-size: 1.1rem; + margin-bottom: 0.5rem; + color: var(--accent-color); +} + +.diagram img, +.diagram object { + width: 100%; + height: auto; + background: white; + border-radius: 4px; +} + +.diagram a { + display: block; + text-align: center; + margin-top: 0.5rem; + color: var(--accent-color); + text-decoration: none; + font-size: 0.9rem; +} + +.diagram a:hover { + text-decoration: underline; +} + +nav { + margin-bottom: 2rem; +} + +nav a { + color: var(--accent-color); + text-decoration: none; + margin-right: 1.5rem; +} + +nav a:hover { + text-decoration: underline; +} + +.legend { + margin-top: 2rem; + padding: 1rem; + background: #252540; + border-radius: 8px; + border: 1px solid var(--border-color); +} + +.legend h3 { + margin-bottom: 0.5rem; +} + +.legend ul { + list-style: none; + display: flex; + flex-wrap: wrap; + gap: 1rem; +} + +.legend li { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.legend .color-box { + width: 16px; + height: 16px; + border-radius: 3px; +} + +code { + background: #333; + padding: 0.2rem 0.4rem; + border-radius: 3px; + font-family: 'Monaco', 'Consolas', monospace; + font-size: 0.9em; +} + +pre { + background: #252540; + padding: 1rem; + border-radius: 8px; + overflow-x: auto; + border: 1px solid var(--border-color); +} + +pre code { + background: none; + padding: 0; +}