better redis error handling

This commit is contained in:
buenosairesam
2025-12-30 03:40:20 -03:00
parent e5aafd5097
commit ee9cbf73ec
2 changed files with 74 additions and 19 deletions

View File

@@ -133,13 +133,16 @@ class MetricsServicer(metrics_pb2_grpc.MetricsServiceServicer):
key = f"{metric_type}:{','.join(f'{k}={v}' for k, v in labels.items())}"
metrics_dict[key] = value
# Update Redis (current state)
await self.redis.update_machine_state(
machine_id=machine_id,
hostname=hostname,
metrics=metrics_dict,
timestamp_ms=timestamp_ms,
)
# Update Redis (current state) - don't fail stream if Redis is down
try:
await self.redis.update_machine_state(
machine_id=machine_id,
hostname=hostname,
metrics=metrics_dict,
timestamp_ms=timestamp_ms,
)
except Exception as e:
self.logger.warning("redis_update_failed", error=str(e))
# Insert into TimescaleDB (historical)
try:
@@ -161,16 +164,19 @@ class MetricsServicer(metrics_pb2_grpc.MetricsServiceServicer):
except Exception as e:
self.logger.warning("machine_registry_update_failed", error=str(e))
# Publish event for subscribers (alerts, gateway)
await self.publisher.publish(
topic="metrics.raw",
payload={
"machine_id": machine_id,
"hostname": hostname,
"timestamp_ms": timestamp_ms,
"metrics": metrics_dict,
},
)
# Publish event for subscribers (alerts, gateway) - don't fail stream
try:
await self.publisher.publish(
topic="metrics.raw",
payload={
"machine_id": machine_id,
"hostname": hostname,
"timestamp_ms": timestamp_ms,
"metrics": metrics_dict,
},
)
except Exception as e:
self.logger.warning("event_publish_failed", error=str(e))
self.logger.debug(
"batch_flushed",