bvandeusenandClaude Opus 5.5
a9bf4b7f16
After Steward or its database was down, every agent's age and every series' last sample were old, so all agent-offline rules fired and all rules went no-data at once, clearing a minute later. With per-account alert email that would have mailed everyone a dozen notices per restart. A periodic job now ticks a Watch (steward/core/watch.py) once it has reached the database; the Watch says since when runs have been unbroken (process start, or the end of a gap over 120 s). seconds_since_seen and the no-data sweep count silence from no earlier than that, so a source must stay quiet while Steward is watching before it counts as silent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
2.9 KiB
Python
75 lines
2.9 KiB
Python
# plugins/host_agent/scheduler.py
|
|
from __future__ import annotations
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from typing import Iterable
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.core.alerts import MetricBatch
|
|
from steward.core.scheduler import ScheduledTask
|
|
from steward.core.watch import watch
|
|
from steward.models.hosts import Host
|
|
from .models import HostAgentRegistration
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
|
|
now: datetime, watching_since: datetime | None = None
|
|
) -> list[tuple[str, float]]:
|
|
"""(host name, seconds since its agent last reported) per registration.
|
|
|
|
An agent that has never reported counts from when it was registered. A
|
|
registration whose host is gone is skipped. Nothing counts from before
|
|
`watching_since`, when Steward last started watching without a break: an
|
|
agent can't report while Steward is down, so that time is not its silence
|
|
(#5441). An agent that is really gone reads as offline once it has been
|
|
quiet that long while Steward watched.
|
|
"""
|
|
out = []
|
|
for r in regs:
|
|
name = host_names.get(r.host_id)
|
|
seen = r.last_seen_at or r.created_at
|
|
if name is None or seen is None:
|
|
continue
|
|
if watching_since is not None:
|
|
seen = max(seen, watching_since)
|
|
out.append((name, max(0.0, (now - seen).total_seconds())))
|
|
return out
|
|
|
|
|
|
async def record_seconds_since_seen(app) -> None:
|
|
"""Record host_agent/<host>/seconds_since_seen for every registered agent,
|
|
so "agent offline" is an ordinary alert rule (seconds_since_seen > 180)."""
|
|
now = datetime.now(timezone.utc)
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
|
|
watching_since = watch(app, "host_agent_seconds_since_seen").tick(now)
|
|
if not regs:
|
|
return
|
|
host_names = dict((await session.execute(
|
|
select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
|
|
)).all())
|
|
batch = MetricBatch()
|
|
for name, seconds in seconds_since_seen(regs, host_names, now, watching_since):
|
|
batch.add(source_module="host_agent", resource_name=name,
|
|
metric_name="seconds_since_seen", value=seconds, recorded_at=now)
|
|
await batch.record(session)
|
|
|
|
|
|
def make_task(app) -> ScheduledTask:
|
|
async def _record():
|
|
try:
|
|
await record_seconds_since_seen(app)
|
|
except Exception:
|
|
logger.exception("host_agent: recording seconds_since_seen failed")
|
|
|
|
return ScheduledTask(
|
|
name="host_agent_seconds_since_seen",
|
|
coro_factory=_record,
|
|
interval_seconds=60,
|
|
run_on_startup=False,
|
|
)
|