better redis error handling
This commit is contained in:
@@ -133,13 +133,16 @@ class MetricsServicer(metrics_pb2_grpc.MetricsServiceServicer):
|
||||
key = f"{metric_type}:{','.join(f'{k}={v}' for k, v in labels.items())}"
|
||||
metrics_dict[key] = value
|
||||
|
||||
# Update Redis (current state)
|
||||
await self.redis.update_machine_state(
|
||||
machine_id=machine_id,
|
||||
hostname=hostname,
|
||||
metrics=metrics_dict,
|
||||
timestamp_ms=timestamp_ms,
|
||||
)
|
||||
# Update Redis (current state) - don't fail stream if Redis is down
|
||||
try:
|
||||
await self.redis.update_machine_state(
|
||||
machine_id=machine_id,
|
||||
hostname=hostname,
|
||||
metrics=metrics_dict,
|
||||
timestamp_ms=timestamp_ms,
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning("redis_update_failed", error=str(e))
|
||||
|
||||
# Insert into TimescaleDB (historical)
|
||||
try:
|
||||
@@ -161,16 +164,19 @@ class MetricsServicer(metrics_pb2_grpc.MetricsServiceServicer):
|
||||
except Exception as e:
|
||||
self.logger.warning("machine_registry_update_failed", error=str(e))
|
||||
|
||||
# Publish event for subscribers (alerts, gateway)
|
||||
await self.publisher.publish(
|
||||
topic="metrics.raw",
|
||||
payload={
|
||||
"machine_id": machine_id,
|
||||
"hostname": hostname,
|
||||
"timestamp_ms": timestamp_ms,
|
||||
"metrics": metrics_dict,
|
||||
},
|
||||
)
|
||||
# Publish event for subscribers (alerts, gateway) - don't fail stream
|
||||
try:
|
||||
await self.publisher.publish(
|
||||
topic="metrics.raw",
|
||||
payload={
|
||||
"machine_id": machine_id,
|
||||
"hostname": hostname,
|
||||
"timestamp_ms": timestamp_ms,
|
||||
"metrics": metrics_dict,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning("event_publish_failed", error=str(e))
|
||||
|
||||
self.logger.debug(
|
||||
"batch_flushed",
|
||||
|
||||
Reference in New Issue
Block a user