simplify woodpecker pipeline, add ctrl/standalone gateway
This commit is contained in:
buenosairesam
189
.woodpecker.yml
189
.woodpecker.yml
@@ -1,184 +1,13 @@
|
||||
# Woodpecker CI Pipeline
|
||||
# https://woodpecker-ci.org/docs/usage/pipeline-syntax
|
||||
# sysmonstm Pipeline
|
||||
|
||||
variables:
|
||||
- &python_image python:3.11-slim
|
||||
- &docker_image docker:24-dind
|
||||
when:
|
||||
- event: push
|
||||
- event: manual
|
||||
|
||||
# Clone settings
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
settings:
|
||||
depth: 50
|
||||
|
||||
# Pipeline steps
|
||||
steps:
|
||||
# ==========================================================================
|
||||
# Lint and Test
|
||||
# ==========================================================================
|
||||
|
||||
lint:
|
||||
image: *python_image
|
||||
- name: notify
|
||||
image: alpine
|
||||
commands:
|
||||
- pip install ruff mypy
|
||||
- ruff check services/ shared/
|
||||
- ruff format --check services/ shared/
|
||||
when:
|
||||
event: [push, pull_request]
|
||||
|
||||
test-shared:
|
||||
image: *python_image
|
||||
commands:
|
||||
- pip install pytest pytest-asyncio redis asyncpg
|
||||
- pip install -r shared/events/requirements.txt || true
|
||||
- pytest shared/ -v --tb=short
|
||||
when:
|
||||
event: [push, pull_request]
|
||||
|
||||
test-services:
|
||||
image: *python_image
|
||||
commands:
|
||||
- pip install pytest pytest-asyncio grpcio grpcio-tools
|
||||
- |
|
||||
for svc in collector aggregator gateway alerts; do
|
||||
if [ -f "services/$svc/requirements.txt" ]; then
|
||||
pip install -r "services/$svc/requirements.txt"
|
||||
fi
|
||||
done
|
||||
- pytest services/ -v --tb=short || true
|
||||
when:
|
||||
event: [push, pull_request]
|
||||
|
||||
# ==========================================================================
|
||||
# Build Docker Images
|
||||
# ==========================================================================
|
||||
|
||||
build-aggregator:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- docker build -t sysmonstm/aggregator:${CI_COMMIT_SHA:0:7} -f services/aggregator/Dockerfile --target production .
|
||||
- docker tag sysmonstm/aggregator:${CI_COMMIT_SHA:0:7} sysmonstm/aggregator:latest
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
when:
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
build-gateway:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- docker build -t sysmonstm/gateway:${CI_COMMIT_SHA:0:7} -f services/gateway/Dockerfile --target production .
|
||||
- docker tag sysmonstm/gateway:${CI_COMMIT_SHA:0:7} sysmonstm/gateway:latest
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
when:
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
build-collector:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- docker build -t sysmonstm/collector:${CI_COMMIT_SHA:0:7} -f services/collector/Dockerfile --target production .
|
||||
- docker tag sysmonstm/collector:${CI_COMMIT_SHA:0:7} sysmonstm/collector:latest
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
when:
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
build-alerts:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- docker build -t sysmonstm/alerts:${CI_COMMIT_SHA:0:7} -f services/alerts/Dockerfile --target production .
|
||||
- docker tag sysmonstm/alerts:${CI_COMMIT_SHA:0:7} sysmonstm/alerts:latest
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
when:
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
# ==========================================================================
|
||||
# Push to Registry
|
||||
# ==========================================================================
|
||||
|
||||
push-images:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- echo "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" --password-stdin "$REGISTRY_URL"
|
||||
- |
|
||||
for img in aggregator gateway collector alerts; do
|
||||
docker tag sysmonstm/$img:latest $REGISTRY_URL/sysmonstm/$img:${CI_COMMIT_SHA:0:7}
|
||||
docker tag sysmonstm/$img:latest $REGISTRY_URL/sysmonstm/$img:latest
|
||||
docker push $REGISTRY_URL/sysmonstm/$img:${CI_COMMIT_SHA:0:7}
|
||||
docker push $REGISTRY_URL/sysmonstm/$img:latest
|
||||
done
|
||||
secrets: [registry_user, registry_password, registry_url]
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
when:
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
# ==========================================================================
|
||||
# Deploy to EC2
|
||||
# ==========================================================================
|
||||
|
||||
deploy-staging:
|
||||
image: appleboy/drone-ssh
|
||||
settings:
|
||||
host:
|
||||
from_secret: deploy_host
|
||||
username:
|
||||
from_secret: deploy_user
|
||||
key:
|
||||
from_secret: deploy_key
|
||||
script:
|
||||
- cd /home/ec2-user/sysmonstm
|
||||
- git pull origin main
|
||||
- docker-compose pull
|
||||
- docker-compose up -d --remove-orphans
|
||||
- docker system prune -f
|
||||
when:
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
# ==========================================================================
|
||||
# Notifications
|
||||
# ==========================================================================
|
||||
|
||||
notify-success:
|
||||
image: plugins/webhook
|
||||
settings:
|
||||
urls:
|
||||
from_secret: webhook_url
|
||||
content_type: application/json
|
||||
template: |
|
||||
{
|
||||
"text": "✅ Build succeeded: ${CI_REPO_NAME}#${CI_BUILD_NUMBER}",
|
||||
"commit": "${CI_COMMIT_SHA:0:7}",
|
||||
"branch": "${CI_COMMIT_BRANCH}",
|
||||
"author": "${CI_COMMIT_AUTHOR}"
|
||||
}
|
||||
when:
|
||||
status: success
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
notify-failure:
|
||||
image: plugins/webhook
|
||||
settings:
|
||||
urls:
|
||||
from_secret: webhook_url
|
||||
content_type: application/json
|
||||
template: |
|
||||
{
|
||||
"text": "❌ Build failed: ${CI_REPO_NAME}#${CI_BUILD_NUMBER}",
|
||||
"commit": "${CI_COMMIT_SHA:0:7}",
|
||||
"branch": "${CI_COMMIT_BRANCH}",
|
||||
"author": "${CI_COMMIT_AUTHOR}"
|
||||
}
|
||||
when:
|
||||
status: failure
|
||||
event: push
|
||||
branch: main
|
||||
- echo "=== sysmonstm ==="
|
||||
- echo "Branch: $CI_COMMIT_BRANCH"
|
||||
- echo "Commit: $CI_COMMIT_SHA"
|
||||
|
||||
6
ctrl/standalone/Dockerfile
Normal file
6
ctrl/standalone/Dockerfile
Normal file
@@ -0,0 +1,6 @@
|
||||
FROM python:3.11-slim
|
||||
WORKDIR /app
|
||||
RUN pip install --no-cache-dir fastapi uvicorn[standard] websockets
|
||||
COPY main.py .
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
32
ctrl/standalone/README.md
Normal file
32
ctrl/standalone/README.md
Normal file
@@ -0,0 +1,32 @@
|
||||
# sysmonstm standalone deployment
|
||||
|
||||
Minimal standalone gateway for AWS deployment without Redis/TimescaleDB dependencies.
|
||||
|
||||
## Status
|
||||
- [x] Created standalone main.py with FastAPI + WebSocket
|
||||
- [x] Created Dockerfile
|
||||
- [x] Created docker-compose.yml (uses gateway network)
|
||||
- [ ] Deploy to AWS
|
||||
- [ ] Update nginx config (sysmonstm.mcrn.ar -> sysmonstm:8080)
|
||||
- [ ] Create local collector script to push metrics
|
||||
|
||||
## Deploy
|
||||
|
||||
```bash
|
||||
# On AWS
|
||||
cd ~/sysmonstm
|
||||
docker compose up -d --build
|
||||
|
||||
# Add to gateway network
|
||||
docker network connect gateway sysmonstm
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
- Gateway shows "no collectors connected" until a collector pushes metrics via WebSocket
|
||||
- Collectors can be run anywhere and connect to wss://sysmonstm.mcrn.ar/ws
|
||||
- No Redis/TimescaleDB needed - metrics stored in memory only
|
||||
|
||||
## TODO
|
||||
- Create simple collector script for local machine
|
||||
- Add basic auth for collector connections
|
||||
13
ctrl/standalone/docker-compose.yml
Normal file
13
ctrl/standalone/docker-compose.yml
Normal file
@@ -0,0 +1,13 @@
|
||||
services:
|
||||
sysmonstm:
|
||||
build: .
|
||||
container_name: sysmonstm
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- gateway
|
||||
|
||||
networks:
|
||||
gateway:
|
||||
external: true
|
||||
198
ctrl/standalone/main.py
Normal file
198
ctrl/standalone/main.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""Minimal sysmonstm gateway - standalone mode without dependencies."""
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import HTMLResponse
|
||||
import json
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
|
||||
app = FastAPI(title="sysmonstm")
|
||||
|
||||
# Store connected websockets
|
||||
connections: list[WebSocket] = []
|
||||
# Store latest metrics from collectors
|
||||
machines: dict = {}
|
||||
|
||||
HTML = """
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>sysmonstm</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #1a1a2e;
|
||||
--bg2: #16213e;
|
||||
--text: #eee;
|
||||
--accent: #e94560;
|
||||
--success: #4ade80;
|
||||
--muted: #666;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
font-family: system-ui, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
min-height: 100vh;
|
||||
padding: 2rem;
|
||||
}
|
||||
header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 2rem;
|
||||
padding-bottom: 1rem;
|
||||
border-bottom: 2px solid var(--accent);
|
||||
}
|
||||
h1 { font-size: 1.5rem; }
|
||||
.status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
.dot {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
background: var(--accent);
|
||||
}
|
||||
.dot.ok { background: var(--success); }
|
||||
.machines {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(350px, 1fr));
|
||||
gap: 1rem;
|
||||
}
|
||||
.machine {
|
||||
background: var(--bg2);
|
||||
border-radius: 8px;
|
||||
padding: 1rem;
|
||||
}
|
||||
.machine h3 { margin-bottom: 0.5rem; }
|
||||
.metric {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 0.25rem 0;
|
||||
border-bottom: 1px solid #333;
|
||||
}
|
||||
.empty {
|
||||
text-align: center;
|
||||
color: var(--muted);
|
||||
padding: 4rem;
|
||||
}
|
||||
.empty p { margin-top: 1rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>sysmonstm</h1>
|
||||
<div class="status">
|
||||
<span class="dot" id="ws-status"></span>
|
||||
<span id="status-text">connecting...</span>
|
||||
</div>
|
||||
</header>
|
||||
<main>
|
||||
<div id="machines" class="machines">
|
||||
<div class="empty">
|
||||
<h2>No collectors connected</h2>
|
||||
<p>Start a collector to see metrics</p>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
<script>
|
||||
const machinesEl = document.getElementById('machines');
|
||||
const statusDot = document.getElementById('ws-status');
|
||||
const statusText = document.getElementById('status-text');
|
||||
let machines = {};
|
||||
|
||||
function connect() {
|
||||
const ws = new WebSocket(`wss://${location.host}/ws`);
|
||||
|
||||
ws.onopen = () => {
|
||||
statusDot.classList.add('ok');
|
||||
statusText.textContent = 'connected';
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
statusDot.classList.remove('ok');
|
||||
statusText.textContent = 'disconnected';
|
||||
setTimeout(connect, 2000);
|
||||
};
|
||||
|
||||
ws.onmessage = (e) => {
|
||||
const data = JSON.parse(e.data);
|
||||
if (data.type === 'metrics') {
|
||||
machines[data.machine_id] = data;
|
||||
render();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function render() {
|
||||
const ids = Object.keys(machines);
|
||||
if (ids.length === 0) {
|
||||
machinesEl.innerHTML = '<div class="empty"><h2>No collectors connected</h2><p>Start a collector to see metrics</p></div>';
|
||||
return;
|
||||
}
|
||||
|
||||
machinesEl.innerHTML = ids.map(id => {
|
||||
const m = machines[id];
|
||||
return `
|
||||
<div class="machine">
|
||||
<h3>${id}</h3>
|
||||
<div class="metric"><span>CPU</span><span>${m.cpu?.toFixed(1) || '-'}%</span></div>
|
||||
<div class="metric"><span>Memory</span><span>${m.memory?.toFixed(1) || '-'}%</span></div>
|
||||
<div class="metric"><span>Disk</span><span>${m.disk?.toFixed(1) || '-'}%</span></div>
|
||||
<div class="metric"><span>Updated</span><span>${new Date(m.timestamp).toLocaleTimeString()}</span></div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
connect();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def index():
|
||||
return HTML
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok", "machines": len(machines)}
|
||||
|
||||
@app.websocket("/ws")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
connections.append(websocket)
|
||||
try:
|
||||
# Send current state
|
||||
for machine_id, data in machines.items():
|
||||
await websocket.send_json({"type": "metrics", "machine_id": machine_id, **data})
|
||||
# Keep alive
|
||||
while True:
|
||||
try:
|
||||
msg = await asyncio.wait_for(websocket.receive_text(), timeout=30)
|
||||
data = json.loads(msg)
|
||||
if data.get("type") == "metrics":
|
||||
machine_id = data.get("machine_id", "unknown")
|
||||
machines[machine_id] = {**data, "timestamp": datetime.utcnow().isoformat()}
|
||||
# Broadcast to all
|
||||
for conn in connections:
|
||||
try:
|
||||
await conn.send_json({"type": "metrics", "machine_id": machine_id, **machines[machine_id]})
|
||||
except:
|
||||
pass
|
||||
except asyncio.TimeoutError:
|
||||
await websocket.send_json({"type": "ping"})
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
connections.remove(websocket)
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8080)
|
||||
Reference in New Issue
Block a user