shoehorning graphql, step functions and lamdas. aws deployment scripts
This commit is contained in:
@@ -3,7 +3,6 @@ digraph job_flow {
|
||||
node [shape=box, style=rounded, fontname="Helvetica"]
|
||||
edge [fontname="Helvetica", fontsize=10]
|
||||
|
||||
// Title
|
||||
labelloc="t"
|
||||
label="MPR - Job Flow"
|
||||
fontsize=16
|
||||
@@ -11,7 +10,19 @@ digraph job_flow {
|
||||
|
||||
graph [splines=ortho, nodesep=0.6, ranksep=0.6]
|
||||
|
||||
// States
|
||||
// API entry points
|
||||
subgraph cluster_api {
|
||||
label="API Entry Points"
|
||||
style=dashed
|
||||
color=gray
|
||||
|
||||
rest_create [label="POST /api/jobs/", shape=ellipse]
|
||||
gql_create [label="mutation createJob", shape=ellipse]
|
||||
rest_cancel [label="POST /api/jobs/{id}/cancel", shape=ellipse]
|
||||
rest_callback [label="POST /api/jobs/{id}/callback", shape=ellipse]
|
||||
}
|
||||
|
||||
// Job states
|
||||
subgraph cluster_states {
|
||||
label="Job States"
|
||||
style=filled
|
||||
@@ -24,78 +35,70 @@ digraph job_flow {
|
||||
cancelled [label="CANCELLED", fillcolor="#6c757d", style="filled,rounded", fontcolor=white]
|
||||
}
|
||||
|
||||
// Transitions
|
||||
pending -> processing [label="worker picks up"]
|
||||
processing -> completed [label="success"]
|
||||
processing -> failed [label="error"]
|
||||
pending -> cancelled [label="user cancels"]
|
||||
processing -> cancelled [label="user cancels"]
|
||||
failed -> pending [label="retry"]
|
||||
// State transitions
|
||||
pending -> processing [xlabel="worker picks up"]
|
||||
processing -> completed [xlabel="success"]
|
||||
processing -> failed [xlabel="error"]
|
||||
pending -> cancelled [xlabel="user cancels"]
|
||||
processing -> cancelled [xlabel="user cancels"]
|
||||
failed -> pending [xlabel="retry"]
|
||||
|
||||
// API actions
|
||||
subgraph cluster_api {
|
||||
label="API Actions"
|
||||
style=dashed
|
||||
color=gray
|
||||
rest_create -> pending
|
||||
gql_create -> pending
|
||||
rest_cancel -> cancelled [style=dashed]
|
||||
|
||||
create_job [label="POST /jobs/", shape=ellipse]
|
||||
cancel_job [label="POST /jobs/{id}/cancel", shape=ellipse]
|
||||
retry_job [label="POST /jobs/{id}/retry", shape=ellipse]
|
||||
}
|
||||
|
||||
create_job -> pending
|
||||
cancel_job -> cancelled [style=dashed]
|
||||
retry_job -> pending [style=dashed]
|
||||
|
||||
// Executor layer
|
||||
subgraph cluster_executor {
|
||||
label="Executor Layer"
|
||||
// Executor dispatch
|
||||
subgraph cluster_dispatch {
|
||||
label="Executor Dispatch"
|
||||
style=filled
|
||||
fillcolor="#fff8e8"
|
||||
|
||||
executor [label="Executor\n(abstract)", shape=diamond]
|
||||
local [label="LocalExecutor\nCelery + FFmpeg"]
|
||||
lambda_exec [label="LambdaExecutor\nSQS + Lambda"]
|
||||
dispatch [label="MPR_EXECUTOR", shape=diamond]
|
||||
}
|
||||
|
||||
processing -> executor
|
||||
executor -> local [label="MPR_EXECUTOR=local"]
|
||||
executor -> lambda_exec [label="MPR_EXECUTOR=lambda", style=dashed]
|
||||
pending -> dispatch
|
||||
|
||||
// FFmpeg operations
|
||||
subgraph cluster_ffmpeg {
|
||||
label="FFmpeg Operations"
|
||||
// Local path
|
||||
subgraph cluster_local {
|
||||
label="Local Mode (Celery)"
|
||||
style=filled
|
||||
fillcolor="#e8f4e8"
|
||||
|
||||
transcode [label="Transcode\n(with preset)"]
|
||||
trim [label="Trim\n(-c:v copy -c:a copy)"]
|
||||
celery_task [label="Celery Task\n(transcode queue)"]
|
||||
s3_download [label="S3 Download\n(MinIO)"]
|
||||
ffmpeg_local [label="FFmpeg\ntranscode/trim"]
|
||||
s3_upload [label="S3 Upload\n(MinIO)"]
|
||||
db_update [label="DB Update\n(update_job_progress)"]
|
||||
}
|
||||
|
||||
local -> transcode
|
||||
local -> trim
|
||||
dispatch -> celery_task [xlabel="local"]
|
||||
celery_task -> s3_download
|
||||
s3_download -> ffmpeg_local
|
||||
ffmpeg_local -> s3_upload
|
||||
s3_upload -> db_update
|
||||
db_update -> completed [style=dotted]
|
||||
|
||||
// gRPC streaming
|
||||
subgraph cluster_grpc {
|
||||
label="gRPC Communication"
|
||||
// Lambda path
|
||||
subgraph cluster_lambda {
|
||||
label="Lambda Mode (AWS)"
|
||||
style=filled
|
||||
fillcolor="#e8e8f8"
|
||||
fillcolor="#fde8d0"
|
||||
|
||||
grpc_stream [label="StreamProgress\n(server streaming)", shape=parallelogram]
|
||||
grpc_submit [label="SubmitJob\n(unary)", shape=parallelogram]
|
||||
grpc_cancel [label="CancelJob\n(unary)", shape=parallelogram]
|
||||
sfn_start [label="Step Functions\nstart_execution"]
|
||||
lambda_fn [label="Lambda\nFFmpeg container"]
|
||||
s3_dl_aws [label="S3 Download\n(AWS)"]
|
||||
ffmpeg_aws [label="FFmpeg\ntranscode/trim"]
|
||||
s3_ul_aws [label="S3 Upload\n(AWS)"]
|
||||
callback [label="HTTP Callback\nPOST /jobs/{id}/callback"]
|
||||
}
|
||||
|
||||
// Progress tracking via gRPC
|
||||
progress [label="Progress Updates\n(gRPC → Redis → DB)", shape=note]
|
||||
transcode -> progress [style=dotted]
|
||||
trim -> progress [style=dotted]
|
||||
progress -> grpc_stream [style=dotted, label="stream to client"]
|
||||
grpc_stream -> processing [style=dotted, label="update status"]
|
||||
dispatch -> sfn_start [xlabel="lambda"]
|
||||
sfn_start -> lambda_fn
|
||||
lambda_fn -> s3_dl_aws
|
||||
s3_dl_aws -> ffmpeg_aws
|
||||
ffmpeg_aws -> s3_ul_aws
|
||||
s3_ul_aws -> callback
|
||||
callback -> completed [style=dotted]
|
||||
|
||||
// gRPC job control
|
||||
create_job -> grpc_submit [label="via gRPC"]
|
||||
grpc_submit -> pending [style=dashed]
|
||||
cancel_job -> grpc_cancel [label="via gRPC"]
|
||||
grpc_cancel -> cancelled [style=dashed]
|
||||
rest_callback -> completed [style=dashed, xlabel="Lambda reports"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user