feat(host_agent): how far each host's clock is from Steward's (milestone 490)

- Agent 1.12.0 stamps each post with sent_at; ingest compares it with the
  moment the post arrived: clock_offset_s / clock_checked_at on the
  registration (host_agent_003), and a clock_off_secs metric (either way)
  that alert rules can watch. Replaces the old skew log, which measured the
  newest sample and was off by up to an interval.
- Settings -> Thresholds: Clock drift, 30 s warn / 5 min critical.
- Host page: 'clock 0.3s ahead' in the vitals line, toned past the warning.
- Hosts list: names hosts whose clock is off; when every reporting host is
  off by about the same amount, says Steward's own clock is the likely one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/formatting.py

@@ -285,6 +285,7 @@ METRICS: dict[str, tuple[str, str]] = {
     "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"),
     "uptime_secs": ("Uptime", "time since the host booted"),
     "seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"),
+    "clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"),
 }
 
 # Where a rule's metric comes from, as the picker and the rules list say it.

steward/core/settings.py

@@ -97,6 +97,9 @@ DEFAULTS: dict[str, Any] = {
     # many container deaths within an hour count as crash-looping.
     "thresholds.tls_warn_days": 14, "thresholds.tls_crit_days": 3,
     "thresholds.crash_loop_deaths": 3,
+    # Seconds a host's clock may be from Steward's before it is called out
+    # (milestone 490), either way.
+    "thresholds.clock_warn_s": 30, "thresholds.clock_crit_s": 300,
     # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s
     # samples are heavy, so keep a short raw window then roll up to hourly
     # averages kept much longer; lifecycle events are light, keep a month.
@@ -448,6 +451,7 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
         "latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"},
         "uptime":  {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"},
         "tls_days": {"warn": g("thresholds.tls_warn_days", 14), "crit": g("thresholds.tls_crit_days", 3), "dir": "low"},
+        "clock":   {"warn": g("thresholds.clock_warn_s", 30), "crit": g("thresholds.clock_crit_s", 300), "dir": "high"},
         "crash_loop": {"warn": g("thresholds.crash_loop_deaths", 3), "crit": g("thresholds.crash_loop_deaths", 3), "dir": "high"},
     }
 

steward/hosts/routes.py

@@ -92,7 +92,10 @@ async def _list_page(add: dict | None = None):
 
     ctx = dict(hosts=shown, total=len(hosts), host_status=host_status, uptime=uptime, agent=agent,
                groups=groups, group_names=group_names, f=f, filter_url=filter_url, sort_url=sort_url,
-               states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add)
+               states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add,
+               clock=host_svc.clock_trouble(
+                   agent, (current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30})["warn"]),
+               host_ids={h.name: h.id for h in hosts})
     # The filter bar's swap gets the table alone. A history restore (Back to a
     # pushed URL) is an HTMX request too but wants the page, so key on the
     # target; Vary keeps the browser from caching one as the other.

steward/hosts/service.py

@@ -249,7 +249,7 @@ async def agent_overview_by_host(
     """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
 
     {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
-                 cores, load_per_core, last_at, fresh}} for each host with any
+                 cores, load_per_core, last_at, fresh, clock_off_secs}} for each host with any
     agent samples. Cores are counted from the per-core CPU sub-resources
     ("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
     of them, the figure Settings → Thresholds grades "load" by.
@@ -266,7 +266,7 @@ async def agent_overview_by_host(
     stale_after = int(stale_after_seconds)
     latest = await latest_by_resource(
         db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
-        set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
+        set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"})
     from steward.models.metrics import PluginMetricLatest
     cores: dict[str, int] = {}
     for res in (await db.execute(
@@ -289,6 +289,8 @@ async def agent_overview_by_host(
         ts = max((r.recorded_at for r in own.values()), default=None)
         d["last_at"] = ts
         d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
+        clock = latest.get(name, {}).get("clock_off_secs")
+        d["clock_off_secs"] = clock.value if clock is not None else None
         d["cores"] = cores.get(name)
         load = d.get("load_1m")
         d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
@@ -296,6 +298,25 @@ async def agent_overview_by_host(
     return out
 
 
+def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None:
+    """Reporting hosts whose clock is more than `limit` seconds from Steward's
+    (Settings → Thresholds' clock drift warning, milestone 490),
+    worst first: {"hosts": [(name, seconds)], "steward": bool, "typical": s}.
+
+    `steward` is set when every reporting host (two or more) is off, and by
+    about the same amount: then the one clock they all disagree with,
+    Steward's own, is the likely culprit. None when every clock is in step."""
+    readings = {n: a["clock_off_secs"] for n, a in agent.items()
+                if a.get("fresh") and a.get("clock_off_secs") is not None}
+    off = {n: v for n, v in readings.items() if v > limit}
+    if not off:
+        return None
+    worst = sorted(off.items(), key=lambda kv: (-kv[1], kv[0]))
+    low, high = min(off.values()), max(off.values())
+    together = len(readings) >= 2 and len(off) == len(readings) and high - low <= max(5.0, 0.2 * low)
+    return {"hosts": worst, "steward": together, "typical": sorted(off.values())[len(off) // 2]}
+
+
 GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5}
 
 

steward/settings/routes.py

@@ -138,6 +138,7 @@ _THRESHOLD_FIELDS = [
     ("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True),
     ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False),
     ("tls_warn", "thresholds.tls_warn_days", False), ("tls_crit", "thresholds.tls_crit_days", False),
+    ("clock_warn", "thresholds.clock_warn_s", False), ("clock_crit", "thresholds.clock_crit_s", False),
     ("crash_loop_deaths", "thresholds.crash_loop_deaths", False),
 ]
 

steward/templates/hosts/list.html

@@ -22,6 +22,14 @@
         {% endif %}
     </div>
 </div>
+{% if clock %}
+{#- Milestone 490: a host whose clock drifts, or Steward's own when every host disagrees with it. #}
+<p class="page-verdict tone-warn">
+  {%- if clock.steward %}Every host's clock is about {{ clock.typical|duration }} off from Steward's: Steward's own clock is the likely one that is wrong. Check what keeps the time in sync on the machine Steward runs on.
+  {%- else %}Clock off from Steward's on {% for name, secs in clock.hosts %}<a href="/hosts/{{ host_ids[name] }}">{{ name }}</a> by {{ secs|duration }}{{ ", " if not loop.last }}{% endfor %}. Check what keeps the time in sync there.
+  {%- endif -%}
+</p>
+{% endif %}
 {% if total %}
 <form class="filter-bar" method="get" action="/hosts/" role="search"
       hx-target="#hosts-table" hx-push-url="true" hx-include="closest form">

steward/templates/settings/thresholds.html

@@ -43,6 +43,7 @@
   {{ pair("Ping latency", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "Round-trip time. Shared with the Ping widget's good/warn colouring.") }}
   {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
   {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }}
+  {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }}
 
   <div class="form-group" style="margin-bottom:0.25rem;">
     <label>Container crash loop <span style="color:var(--text-muted);font-size:0.8rem;">(deaths per hour)</span></label>