FabledSteward/plugins/host_agent/__init__.py

# plugins/host_agent/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from quart import Quart

_app: "Quart | None" = None


def setup(app: "Quart") -> None:
    global _app
    _app = app
    # A host's CPU, memory and load per core learn their normal (#5742), each
    # with its floor from Settings → Thresholds. Parts don't: a host's CPU has
    # a normal, each of its cores doesn't. Disks stay on their fixed cutoffs:
    # a nearly full disk is a failure whatever its history (System 182).
    from steward.core.norms import floor_setting, learn_normal
    learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False)
    learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False)
    learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False)
    # "Agent offline" speaks for everything the agent carries: while it is
    # firing, the host's other rules going silent isn't emailed (#5721).
    from steward.core.alerts import agent_fed, agent_heartbeat
    from steward.hosts.service import agent_stale_after_seconds
    agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config))
    agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
    # A reboot the agent announced is silence with a reason, for a while (#5756).
    from datetime import datetime, timezone
    from steward.core.alerts import agents_away
    from .downtime import hosts_away, restart_grace_seconds
    agents_away(lambda session: hosts_away(session, datetime.now(timezone.utc),
                                           restart_grace_seconds(app.config)))
    from steward.core.cleanup import prune_by_age
    from .models import HostDowntime
    prune_by_age(HostDowntime.started_at)


def get_audit_entities() -> dict:
    """Audit rows about a maintenance plan name it and link to the plans page."""
    from .models import HostMaintenancePlan
    return {"maintenance_plan": (HostMaintenancePlan, "/plugins/host_agent/maintenance")}


def get_scheduled_tasks() -> list:
    from .scheduler import make_maintenance_task, make_sensor_task, make_task
    return [make_task(_app), make_maintenance_task(_app), make_sensor_task(_app)]


def get_blueprint():
    from .routes import host_agent_bp
    return host_agent_bp


def register_mcp_tools(register) -> None:
    from .mcp_tools import register_tools
    register_tools(register)


async def mcp_overview(db) -> list[dict]:
    from .mcp_tools import mcp_overview as findings
    return await findings(db)


async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
    from .mcp_tools import mcp_events as events
    return await events(db, host_id, host_name, since, until)


def get_alert_metrics() -> dict[str, list[str]]:
    """Host metrics ingest records for alert rules (resource = the host name;
    disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
    comes from the scheduler, not the agent: "agent offline" is > 180.
    clock_off_secs is how far the host's clock is from Steward's, either way
    (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
    of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
    disk_used_pct_worst is the fullest mount by space or inodes.
    mounts_stalled counts the mounts that didn't answer the agent in time
    (agent ≥ 1.14.0): "an NFS share stopped" is > 0. agent_rss_bytes and
    agent_cpu_pct are the agent's own memory and CPU (agent ≥ 1.15.0): a
    healthy agent sits in tens of MB and a few %. updates_pending and
    updates_security are what the package manager has waiting, and
    reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
    "security updates waiting" is updates_security > 0. maintenance_failed
    comes from the scheduler (#5768): 1 when the host's latest maintenance job
    failed or timed out, 0 once one succeeds: "maintenance failed" is > 0.
    temp_c_max is the host's hottest real thermometer, in °C (#5787); sensors
    with nothing wired to them are left out (#5759, #5783): "running hot" is
    > 85. temp_c is one sensor, resource "<host>:temp:<label>". net_rx_bps /
    net_tx_bps and disk_read_bps / disk_write_bps are bytes per second, for
    the whole host ("<host>") or one interface ("<host>:net:<iface>") or disk
    ("<host>:diskio:<device>").

    NFS delay (agent ≥ 1.22.0, #5996), per NFS mount on "<host>:<mount>",
    over the requests it finished since the last sample: nfs_exe_ms is the
    average time a program waited on one (queueing included), nfs_rtt_ms the
    server's round trip, nfs_read_ms / nfs_write_ms the same for reads and
    writes alone, nfs_ops_per_sec how many, and nfs_retrans / nfs_timeouts the
    retransmissions and major timeouts among them; nfs_tx_bps / nfs_rx_bps
    (agent ≥ 1.25.0, #6115) are the bytes a second it sent and received, its
    share of the host's network traffic. nfs_exe_max_ms is the
    host's slowest mount, for a covering rule: "NFS slow" is > 1000 for 3
    checks. A healthy share answers in a few ms.

    Drive health (agent ≥ 1.20.0, #5789), read hourly with smartctl by the
    root helper and judged in drives.py, per drive on "<host>:drive:<device>":
    drive_failing is 1 when the drive fails its own self-assessment, raises
    an NVMe critical warning or runs out of spare blocks; drive_life_used_pct
    is how much of its rated life it has used; drive_bad_sectors counts its
    media errors or reallocated / pending / uncorrectable sectors;
    drive_temp_c is its own temperature. Per host, for covering rules:
    drives_failing (how many) and drive_life_used_max_pct (the most worn):
    "drive failing" is drives_failing > 0, "drive wearing out" is
    drive_life_used_max_pct > 90.

    Recorded but not offered (NOT_ALERTABLE): disk_used_bytes and
    disk_total_bytes, where disk_used_pct is the figure to watch (and docker
    uses disk_total_bytes for something else), and mem_cached_bytes /
    mem_buffers_bytes, parts of what mem_available_bytes already counts."""
    from .drives import DRIVE_METRICS
    return {"host_agent": [
        "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
        "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
        "load_pct",
        "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
        "agent_rss_bytes", "agent_cpu_pct",
        "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
        "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
        "nfs_exe_max_ms", "nfs_exe_ms", "nfs_rtt_ms", "nfs_read_ms", "nfs_write_ms",
        "nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts", "nfs_tx_bps", "nfs_rx_bps",
        *DRIVE_METRICS,
    ]}


# Metrics host_agent records that no rule is offered, each for the reason
# get_alert_metrics gives; tests/core/test_metric_catalog.py holds every other
# recorded name to the catalog.
NOT_ALERTABLE = frozenset({"disk_used_bytes", "disk_total_bytes", "mem_cached_bytes", "mem_buffers_bytes"})