bvandeusenandClaude Opus 5.5
3f9aae7f46
Each sample carries each NFS mount's requests since the last one: how many
a second, their round trip and execute time (READ and WRITE on their own),
and how many were sent again or hit a major timeout. mountstats is the
kernel's own count and never touches the share, so it reads while one is
stuck. Steward stores them per mount and the slowest on the host as
nfs_exe_max_ms, for an "NFS slow" rule and the host page's NFS chart beside
mounts_stalled; each mount's bar says what a request took.
Agent 1.22.0 (which also carries the backlog order from 41fc807).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
137 lines
6.9 KiB
Python
137 lines
6.9 KiB
Python
# plugins/host_agent/__init__.py
|
|
from __future__ import annotations
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
_app: "Quart | None" = None
|
|
|
|
|
|
def setup(app: "Quart") -> None:
|
|
global _app
|
|
_app = app
|
|
# A host's CPU, memory and load per core learn their normal (#5742), each
|
|
# with its floor from Settings → Thresholds. Parts don't: a host's CPU has
|
|
# a normal, each of its cores doesn't. Disks stay on their fixed cutoffs:
|
|
# a nearly full disk is a failure whatever its history (System 182).
|
|
from steward.core.norms import floor_setting, learn_normal
|
|
learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False)
|
|
learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False)
|
|
learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False)
|
|
# "Agent offline" speaks for everything the agent carries: while it is
|
|
# firing, the host's other rules going silent isn't emailed (#5721).
|
|
from steward.core.alerts import agent_fed, agent_heartbeat
|
|
from steward.hosts.service import agent_stale_after_seconds
|
|
agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config))
|
|
agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
|
|
# A reboot the agent announced is silence with a reason, for a while (#5756).
|
|
from datetime import datetime, timezone
|
|
from steward.core.alerts import agents_away
|
|
from .downtime import hosts_away, restart_grace_seconds
|
|
agents_away(lambda session: hosts_away(session, datetime.now(timezone.utc),
|
|
restart_grace_seconds(app.config)))
|
|
from steward.core.cleanup import prune_by_age
|
|
from .models import HostDowntime
|
|
prune_by_age(HostDowntime.started_at)
|
|
|
|
|
|
def get_audit_entities() -> dict:
|
|
"""Audit rows about a maintenance plan name it and link to the plans page."""
|
|
from .models import HostMaintenancePlan
|
|
return {"maintenance_plan": (HostMaintenancePlan, "/plugins/host_agent/maintenance")}
|
|
|
|
|
|
def get_scheduled_tasks() -> list:
|
|
from .scheduler import make_maintenance_task, make_sensor_task, make_task
|
|
return [make_task(_app), make_maintenance_task(_app), make_sensor_task(_app)]
|
|
|
|
|
|
def get_blueprint():
|
|
from .routes import host_agent_bp
|
|
return host_agent_bp
|
|
|
|
|
|
def register_mcp_tools(register) -> None:
|
|
from .mcp_tools import register_tools
|
|
register_tools(register)
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
from .mcp_tools import mcp_overview as findings
|
|
return await findings(db)
|
|
|
|
|
|
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
|
|
from .mcp_tools import mcp_events as events
|
|
return await events(db, host_id, host_name, since, until)
|
|
|
|
|
|
def get_alert_metrics() -> dict[str, list[str]]:
|
|
"""Host metrics ingest records for alert rules (resource = the host name;
|
|
disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
|
|
comes from the scheduler, not the agent: "agent offline" is > 180.
|
|
clock_off_secs is how far the host's clock is from Steward's, either way
|
|
(agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
|
|
of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
|
|
disk_used_pct_worst is the fullest mount by space or inodes.
|
|
mounts_stalled counts the mounts that didn't answer the agent in time
|
|
(agent ≥ 1.14.0): "an NFS share stopped" is > 0. agent_rss_bytes and
|
|
agent_cpu_pct are the agent's own memory and CPU (agent ≥ 1.15.0): a
|
|
healthy agent sits in tens of MB and a few %. updates_pending and
|
|
updates_security are what the package manager has waiting, and
|
|
reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
|
|
"security updates waiting" is updates_security > 0. maintenance_failed
|
|
comes from the scheduler (#5768): 1 when the host's latest maintenance job
|
|
failed or timed out, 0 once one succeeds: "maintenance failed" is > 0.
|
|
temp_c_max is the host's hottest real thermometer, in °C (#5787); sensors
|
|
with nothing wired to them are left out (#5759, #5783): "running hot" is
|
|
> 85. temp_c is one sensor, resource "<host>:temp:<label>". net_rx_bps /
|
|
net_tx_bps and disk_read_bps / disk_write_bps are bytes per second, for
|
|
the whole host ("<host>") or one interface ("<host>:net:<iface>") or disk
|
|
("<host>:diskio:<device>").
|
|
|
|
NFS delay (agent ≥ 1.22.0, #5996), per NFS mount on "<host>:<mount>",
|
|
over the requests it finished since the last sample: nfs_exe_ms is the
|
|
average time a program waited on one (queueing included), nfs_rtt_ms the
|
|
server's round trip, nfs_read_ms / nfs_write_ms the same for reads and
|
|
writes alone, nfs_ops_per_sec how many, and nfs_retrans / nfs_timeouts the
|
|
retransmissions and major timeouts among them. nfs_exe_max_ms is the
|
|
host's slowest mount, for a covering rule: "NFS slow" is > 1000 for 3
|
|
checks. A healthy share answers in a few ms.
|
|
|
|
Drive health (agent ≥ 1.20.0, #5789), read hourly with smartctl by the
|
|
root helper and judged in drives.py, per drive on "<host>:drive:<device>":
|
|
drive_failing is 1 when the drive fails its own self-assessment, raises
|
|
an NVMe critical warning or runs out of spare blocks; drive_life_used_pct
|
|
is how much of its rated life it has used; drive_bad_sectors counts its
|
|
media errors or reallocated / pending / uncorrectable sectors;
|
|
drive_temp_c is its own temperature. Per host, for covering rules:
|
|
drives_failing (how many) and drive_life_used_max_pct (the most worn):
|
|
"drive failing" is drives_failing > 0, "drive wearing out" is
|
|
drive_life_used_max_pct > 90.
|
|
|
|
Recorded but not offered (NOT_ALERTABLE): disk_used_bytes and
|
|
disk_total_bytes, where disk_used_pct is the figure to watch (and docker
|
|
uses disk_total_bytes for something else), and mem_cached_bytes /
|
|
mem_buffers_bytes, parts of what mem_available_bytes already counts."""
|
|
from .drives import DRIVE_METRICS
|
|
return {"host_agent": [
|
|
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
|
|
"disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
|
|
"load_pct",
|
|
"uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
|
|
"agent_rss_bytes", "agent_cpu_pct",
|
|
"updates_pending", "updates_security", "reboot_required", "maintenance_failed",
|
|
"temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
|
|
"nfs_exe_max_ms", "nfs_exe_ms", "nfs_rtt_ms", "nfs_read_ms", "nfs_write_ms",
|
|
"nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts",
|
|
*DRIVE_METRICS,
|
|
]}
|
|
|
|
|
|
# Metrics host_agent records that no rule is offered, each for the reason
|
|
# get_alert_metrics gives; tests/core/test_metric_catalog.py holds every other
|
|
# recorded name to the catalog.
|
|
NOT_ALERTABLE = frozenset({"disk_used_bytes", "disk_total_bytes", "mem_cached_bytes", "mem_buffers_bytes"})
|