feat(mcp): docker and host-agent tools; TLS and crash-loop cutoffs in Settings
MCP-5 of milestone 461 "MCP server: Steward as an LLM-inspectable, LLM-operable hub". Contributed through each plugin's register_mcp_tools hook, so they exist only while the plugin is enabled. docker (viewer, read-only): - list_containers: filter by host, status, compose project, or problem (restarting, crash_looping, unhealthy, oom_killed, exited_nonzero); problems sort first and each row names what is wrong. - get_container: state, CPU/memory history across the raw/hourly seam, lifecycle events. A unique part of a swarm task name is enough. - list_swarm_services: replicas per node (agent-less placements as ghosts) and the swarm's nodes. - search_container_logs: text (% and _ literal), stream, range, context lines, paging by a full-precision `before` cursor; over-long lines keep both ends and say how much of the middle was cut. - get_docker_disk_usage: bytes per host, reclaimable first. - mcp_overview: failing containers and under-replicated swarm services join get_overview. host_agent: list_agents (reporting / stale / pending, version, OS), and an overview finding for agents registered over 30 minutes ago that never checked in. Settings > Thresholds gains the TLS-expiry warn/crit days (were 14/3 in code) and the crash-loop deaths-per-hour count (3), which get_overview and the docker tools now read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -79,6 +79,10 @@ DEFAULTS: dict[str, Any] = {
|
||||
"thresholds.load_warn": 80, "thresholds.load_crit": 100,
|
||||
"thresholds.temp_warn": 70, "thresholds.temp_crit": 85,
|
||||
"thresholds.uptime_warn": 99.0, "thresholds.uptime_crit": 95.0,
|
||||
# Days left on an HTTPS monitor's certificate (lower is worse), and how
|
||||
# many container deaths within an hour count as crash-looping.
|
||||
"thresholds.tls_warn_days": 14, "thresholds.tls_crit_days": 3,
|
||||
"thresholds.crash_loop_deaths": 3,
|
||||
# Docker time-series retention (rule 25 — tunable, no restart). Raw 30s
|
||||
# samples are heavy, so keep a short raw window then roll up to hourly
|
||||
# averages kept much longer; lifecycle events are light, keep a month.
|
||||
@@ -379,6 +383,8 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
|
||||
"temp": {"warn": g("thresholds.temp_warn", 70), "crit": g("thresholds.temp_crit", 85), "dir": "high"},
|
||||
"latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"},
|
||||
"uptime": {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"},
|
||||
"tls_days": {"warn": g("thresholds.tls_warn_days", 14), "crit": g("thresholds.tls_crit_days", 3), "dir": "low"},
|
||||
"crash_loop": {"warn": g("thresholds.crash_loop_deaths", 3), "crit": g("thresholds.crash_loop_deaths", 3), "dir": "high"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ from ._shape import age_s, db_session, iso, now_utc, rnd
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TLS_WARN_DAYS = 14
|
||||
FINDINGS_CAP = 50
|
||||
_SEVERITY = {"critical": 0, "warning": 1}
|
||||
|
||||
@@ -39,7 +38,11 @@ def rank(findings: list[dict]) -> list[dict]:
|
||||
-(f.get("age_s") or 0), f.get("subject", "")))
|
||||
|
||||
|
||||
def monitor_findings(monitors, states: dict, host_names: dict, now) -> list[dict]:
|
||||
def monitor_findings(monitors, states: dict, host_names: dict, now,
|
||||
tls: dict | None = None) -> list[dict]:
|
||||
"""Failing monitors, and HTTPS certificates inside the TLS window
|
||||
(`tls` is THRESHOLDS["tls_days"]: warn/crit days left)."""
|
||||
tls = tls or {"warn": 14, "crit": 3}
|
||||
out = []
|
||||
for m in monitors:
|
||||
if not m.enabled:
|
||||
@@ -58,9 +61,9 @@ def monitor_findings(monitors, states: dict, host_names: dict, now) -> list[dict
|
||||
next=f'get_monitor(monitor="{m.id}")'))
|
||||
elif latest.tls_expires_at is not None:
|
||||
days = (latest.tls_expires_at - now).total_seconds() / 86400.0
|
||||
if days <= TLS_WARN_DAYS:
|
||||
if days <= tls["warn"]:
|
||||
out.append(finding(
|
||||
"critical" if days <= 3 else "warning", "tls_expiring", m.name,
|
||||
"critical" if days <= tls["crit"] else "warning", "tls_expiring", m.name,
|
||||
f"TLS certificate for {m.target} expires in {days:.1f} days",
|
||||
ids=ids, next=f'get_monitor(monitor="{m.id}")'))
|
||||
return out
|
||||
@@ -151,9 +154,9 @@ async def get_overview() -> dict:
|
||||
|
||||
Returns `findings` ranked critical first, then longest-standing first:
|
||||
failing monitors (with the error and since when), firing, pending and
|
||||
still-breaching acknowledged alerts, TLS certificates expiring within
|
||||
14 days, host agents that stopped reporting, hosts over the CPU, memory or
|
||||
disk thresholds set under Settings → Thresholds, Ansible runs that failed
|
||||
still-breaching acknowledged alerts, TLS certificates near expiry, host
|
||||
agents that stopped reporting, hosts over their CPU, memory or disk
|
||||
limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
|
||||
in the last 24h, and anything enabled plugins add (unhealthy containers,
|
||||
for one). Each finding carries the ids to follow and `next`, the call
|
||||
that shows its evidence.
|
||||
@@ -180,9 +183,10 @@ async def get_overview() -> dict:
|
||||
)).scalars())
|
||||
windows = await alert_svc.list_maintenance_windows(db)
|
||||
|
||||
findings = (monitor_findings(monitors, states, host_names, now)
|
||||
thresholds = app.config.get("THRESHOLDS") or {}
|
||||
findings = (monitor_findings(monitors, states, host_names, now, thresholds.get("tls_days"))
|
||||
+ alert_findings(active)
|
||||
+ agent_findings(hosts, agent, app.config.get("THRESHOLDS") or {})
|
||||
+ agent_findings(hosts, agent, thresholds)
|
||||
+ run_findings(runs))
|
||||
plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
|
||||
findings = rank(findings + plugin_found)
|
||||
|
||||
@@ -126,6 +126,8 @@ _THRESHOLD_FIELDS = [
|
||||
("temp_warn", "thresholds.temp_warn", False), ("temp_crit", "thresholds.temp_crit", False),
|
||||
("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True),
|
||||
("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False),
|
||||
("tls_warn", "thresholds.tls_warn_days", False), ("tls_crit", "thresholds.tls_crit_days", False),
|
||||
("crash_loop_deaths", "thresholds.crash_loop_deaths", False),
|
||||
]
|
||||
|
||||
# Docker time-series retention windows (days). Read fresh by the cleanup task,
|
||||
|
||||
@@ -40,6 +40,18 @@
|
||||
{{ pair("Temperature", "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }}
|
||||
{{ pair("Ping latency", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "Round-trip time. Shared with the Ping widget's good/warn colouring.") }}
|
||||
{{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
|
||||
{{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }}
|
||||
|
||||
<div class="form-group" style="margin-bottom:0.25rem;">
|
||||
<label>Container crash loop <span style="color:var(--text-muted);font-size:0.8rem;">(deaths per hour)</span></label>
|
||||
<div style="margin-top:0.25rem;">
|
||||
<input type="number" name="crash_loop_deaths" min="1" step="1"
|
||||
value="{{ settings['thresholds.crash_loop_deaths'] }}" style="max-width:110px;">
|
||||
</div>
|
||||
<div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">
|
||||
A container that dies (or is OOM-killed) this many times within an hour is reported as crash-looping.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% macro days(label, field, key, hint) %}
|
||||
|
||||
Reference in New Issue
Block a user