bvandeusenandClaude Opus 5.5
26a3ec3339
temp_c_max was recorded on every ingest, but host_agent didn't offer it to rules, so "host running hot" couldn't be watched. The catalog guard didn't see the row(...) helper ingest builds its rows with. Now offered: temp_c_max, temp_c per sensor, and network and disk throughput per host, interface and disk. A new guard holds every literal row(...) name to the catalog, except NOT_ALERTABLE, where each exclusion is given a reason. temp_c_max reads in °C. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
108 lines
5.3 KiB
Python
108 lines
5.3 KiB
Python
# plugins/host_agent/__init__.py
|
|
from __future__ import annotations
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
_app: "Quart | None" = None
|
|
|
|
|
|
def setup(app: "Quart") -> None:
|
|
global _app
|
|
_app = app
|
|
# A host's CPU, memory and load per core learn their normal (#5742), each
|
|
# with its floor from Settings → Thresholds. Parts don't: a host's CPU has
|
|
# a normal, each of its cores doesn't. Disks stay on their fixed cutoffs:
|
|
# a nearly full disk is a failure whatever its history (System 182).
|
|
from steward.core.norms import floor_setting, learn_normal
|
|
learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False)
|
|
learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False)
|
|
learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False)
|
|
# "Agent offline" speaks for everything the agent carries: while it is
|
|
# firing, the host's other rules going silent isn't emailed (#5721).
|
|
from steward.core.alerts import agent_fed, agent_heartbeat
|
|
from steward.hosts.service import agent_stale_after_seconds
|
|
agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config))
|
|
agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
|
|
# A reboot the agent announced is silence with a reason, for a while (#5756).
|
|
from datetime import datetime, timezone
|
|
from steward.core.alerts import agents_away
|
|
from .downtime import hosts_away, restart_grace_seconds
|
|
agents_away(lambda session: hosts_away(session, datetime.now(timezone.utc),
|
|
restart_grace_seconds(app.config)))
|
|
from steward.core.cleanup import prune_by_age
|
|
from .models import HostDowntime
|
|
prune_by_age(HostDowntime.started_at)
|
|
|
|
|
|
def get_audit_entities() -> dict:
|
|
"""Audit rows about a maintenance plan name it and link to the plans page."""
|
|
from .models import HostMaintenancePlan
|
|
return {"maintenance_plan": (HostMaintenancePlan, "/plugins/host_agent/maintenance")}
|
|
|
|
|
|
def get_scheduled_tasks() -> list:
|
|
from .scheduler import make_maintenance_task, make_sensor_task, make_task
|
|
return [make_task(_app), make_maintenance_task(_app), make_sensor_task(_app)]
|
|
|
|
|
|
def get_blueprint():
|
|
from .routes import host_agent_bp
|
|
return host_agent_bp
|
|
|
|
|
|
def register_mcp_tools(register) -> None:
|
|
from .mcp_tools import register_tools
|
|
register_tools(register)
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
from .mcp_tools import mcp_overview as findings
|
|
return await findings(db)
|
|
|
|
|
|
def get_alert_metrics() -> dict[str, list[str]]:
|
|
"""Host metrics ingest records for alert rules (resource = the host name;
|
|
disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
|
|
comes from the scheduler, not the agent: "agent offline" is > 180.
|
|
clock_off_secs is how far the host's clock is from Steward's, either way
|
|
(agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
|
|
of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
|
|
disk_used_pct_worst is the fullest mount by space or inodes.
|
|
mounts_stalled counts the mounts that didn't answer the agent in time
|
|
(agent ≥ 1.14.0): "an NFS share stopped" is > 0. agent_rss_bytes and
|
|
agent_cpu_pct are the agent's own memory and CPU (agent ≥ 1.15.0): a
|
|
healthy agent sits in tens of MB and a few %. updates_pending and
|
|
updates_security are what the package manager has waiting, and
|
|
reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
|
|
"security updates waiting" is updates_security > 0. maintenance_failed
|
|
comes from the scheduler (#5768): 1 when the host's latest maintenance job
|
|
failed or timed out, 0 once one succeeds: "maintenance failed" is > 0.
|
|
temp_c_max is the host's hottest real thermometer, in °C (#5787); sensors
|
|
with nothing wired to them are left out (#5759, #5783): "running hot" is
|
|
> 85. temp_c is one sensor, resource "<host>:temp:<label>". net_rx_bps /
|
|
net_tx_bps and disk_read_bps / disk_write_bps are bytes per second, for
|
|
the whole host ("<host>") or one interface ("<host>:net:<iface>") or disk
|
|
("<host>:diskio:<device>").
|
|
|
|
Recorded but not offered (NOT_ALERTABLE): disk_used_bytes and
|
|
disk_total_bytes, where disk_used_pct is the figure to watch (and docker
|
|
uses disk_total_bytes for something else), and mem_cached_bytes /
|
|
mem_buffers_bytes, parts of what mem_available_bytes already counts."""
|
|
return {"host_agent": [
|
|
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
|
|
"disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
|
|
"load_pct",
|
|
"uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
|
|
"agent_rss_bytes", "agent_cpu_pct",
|
|
"updates_pending", "updates_security", "reboot_required", "maintenance_failed",
|
|
"temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
|
|
]}
|
|
|
|
|
|
# Metrics host_agent records that no rule is offered, each for the reason
|
|
# get_alert_metrics gives; tests/core/test_metric_catalog.py holds every other
|
|
# recorded name to the catalog.
|
|
NOT_ALERTABLE = frozenset({"disk_used_bytes", "disk_total_bytes", "mem_cached_bytes", "mem_buffers_bytes"})
|