feat(alerts): one evaluation path; host-agent ingest fires rules (#5060)
#5060 "One evaluation path: record_metrics batch API; host-agent ingest evaluates rules". - record_metrics(session, rows) writes metric rows and evaluates the rules that watch them, reading rules, states and maintenance windows once per batch. Each rule sees its key's samples oldest first, so an agent catching up after an outage counts consecutive checks as they happened. - Host-agent ingest now goes through it. Before, it wrote plugin_metrics directly and no host_agent rule could ever fire. The catalog gains the per-mount disk_used_pct ("<host>:<mount>"). - MetricBatch gathers a transaction's samples; the docker ingest, traefik, unifi, snmp, monitor and self-monitor writers record one batch each instead of a rule query per sample. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -266,7 +266,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
log batches (m79) — appended one row per line when present, independent of
|
||||
whether this sample also carried container metrics.
|
||||
"""
|
||||
from steward.core.alerts import record_metric
|
||||
from steward.core.alerts import MetricBatch
|
||||
from .models import DockerContainer, DockerEvent, DockerMetric
|
||||
|
||||
if swarm is not None:
|
||||
@@ -313,6 +313,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
))
|
||||
|
||||
# Current state + alerts from the newest snapshot only.
|
||||
batch = MetricBatch()
|
||||
for c in latest_containers:
|
||||
name = c.get("name")
|
||||
if not name:
|
||||
@@ -350,19 +351,19 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
# on different hosts don't collide in the metric/alert namespace.
|
||||
resource = f"{host.name}/{name}"
|
||||
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
||||
await record_metric(
|
||||
session=session, source_module="docker",
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
|
||||
)
|
||||
if c.get("mem_pct") is not None:
|
||||
await record_metric(
|
||||
session=session, source_module="docker",
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
|
||||
)
|
||||
# Restart count is alertable regardless of state (crash-looping matters
|
||||
# most while the container is down/restarting).
|
||||
await record_metric(
|
||||
session=session, source_module="docker",
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="restart_count",
|
||||
value=float(c.get("restart_count", 0) or 0),
|
||||
)
|
||||
@@ -371,12 +372,14 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
# container).
|
||||
health = c.get("health")
|
||||
if health in ("healthy", "unhealthy", "starting"):
|
||||
await record_metric(
|
||||
session=session, source_module="docker",
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="is_healthy",
|
||||
value=1.0 if health == "healthy" else 0.0,
|
||||
)
|
||||
|
||||
await batch.record(session)
|
||||
|
||||
# Reap containers that vanished from the host's latest listing. Without this,
|
||||
# removed one-shot containers (CI job runners, buildkit builders, codex jobs)
|
||||
# accumulate forever as permanent "stopped" rows and inflate every count.
|
||||
|
||||
Reference in New Issue
Block a user