feat(host_agent): how far each host's clock is from Steward's (milestone 490)

- Agent 1.12.0 stamps each post with sent_at; ingest compares it with the
  moment the post arrived: clock_offset_s / clock_checked_at on the
  registration (host_agent_003), and a clock_off_secs metric (either way)
  that alert rules can watch. Replaces the old skew log, which measured the
  newest sample and was off by up to an interval.
- Settings -> Thresholds: Clock drift, 30 s warn / 5 min critical.
- Host page: 'clock 0.3s ahead' in the vitals line, toned past the warning.
- Hosts list: names hosts whose clock is off; when every reporting host is
  off by about the same amount, says Steward's own clock is the likely one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/hosts/routes.py

@@ -92,7 +92,10 @@ async def _list_page(add: dict | None = None):
 
     ctx = dict(hosts=shown, total=len(hosts), host_status=host_status, uptime=uptime, agent=agent,
                groups=groups, group_names=group_names, f=f, filter_url=filter_url, sort_url=sort_url,
-               states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add)
+               states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add,
+               clock=host_svc.clock_trouble(
+                   agent, (current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30})["warn"]),
+               host_ids={h.name: h.id for h in hosts})
     # The filter bar's swap gets the table alone. A history restore (Back to a
     # pushed URL) is an HTMX request too but wants the page, so key on the
     # target; Vary keeps the browser from caching one as the other.

steward/hosts/service.py

@@ -249,7 +249,7 @@ async def agent_overview_by_host(
     """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
 
     {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
-                 cores, load_per_core, last_at, fresh}} for each host with any
+                 cores, load_per_core, last_at, fresh, clock_off_secs}} for each host with any
     agent samples. Cores are counted from the per-core CPU sub-resources
     ("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
     of them, the figure Settings → Thresholds grades "load" by.
@@ -266,7 +266,7 @@ async def agent_overview_by_host(
     stale_after = int(stale_after_seconds)
     latest = await latest_by_resource(
         db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
-        set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
+        set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"})
     from steward.models.metrics import PluginMetricLatest
     cores: dict[str, int] = {}
     for res in (await db.execute(
@@ -289,6 +289,8 @@ async def agent_overview_by_host(
         ts = max((r.recorded_at for r in own.values()), default=None)
         d["last_at"] = ts
         d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
+        clock = latest.get(name, {}).get("clock_off_secs")
+        d["clock_off_secs"] = clock.value if clock is not None else None
         d["cores"] = cores.get(name)
         load = d.get("load_1m")
         d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
@@ -296,6 +298,25 @@ async def agent_overview_by_host(
     return out
 
 
+def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None:
+    """Reporting hosts whose clock is more than `limit` seconds from Steward's
+    (Settings → Thresholds' clock drift warning, milestone 490),
+    worst first: {"hosts": [(name, seconds)], "steward": bool, "typical": s}.
+
+    `steward` is set when every reporting host (two or more) is off, and by
+    about the same amount: then the one clock they all disagree with,
+    Steward's own, is the likely culprit. None when every clock is in step."""
+    readings = {n: a["clock_off_secs"] for n, a in agent.items()
+                if a.get("fresh") and a.get("clock_off_secs") is not None}
+    off = {n: v for n, v in readings.items() if v > limit}
+    if not off:
+        return None
+    worst = sorted(off.items(), key=lambda kv: (-kv[1], kv[0]))
+    low, high = min(off.values()), max(off.values())
+    together = len(readings) >= 2 and len(off) == len(readings) and high - low <= max(5.0, 0.2 * low)
+    return {"hosts": worst, "steward": together, "typical": sorted(off.values())[len(off) // 2]}
+
+
 GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5}