feat(host_agent): how far each host's clock is from Steward's (milestone 490)

- Agent 1.12.0 stamps each post with sent_at; ingest compares it with the
  moment the post arrived: clock_offset_s / clock_checked_at on the
  registration (host_agent_003), and a clock_off_secs metric (either way)
  that alert rules can watch. Replaces the old skew log, which measured the
  newest sample and was off by up to an interval.
- Settings -> Thresholds: Clock drift, 30 s warn / 5 min critical.
- Host page: 'clock 0.3s ahead' in the vitals line, toned past the warning.
- Hosts list: names hosts whose clock is off; when every reporting host is
  off by about the same amount, says Steward's own clock is the likely one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/routes.py

@@ -44,6 +44,35 @@ def _parse_ts(ts: str) -> datetime:
     return datetime.fromisoformat(ts)
 
 
+def clock_offset(sent_at, received_at: datetime) -> float | None:
+    """Seconds this host's clock is ahead of Steward's (negative: behind),
+    from the agent's sent_at and the moment Steward received the post. It
+    carries the request's own latency, well under a second on a LAN. None
+    when the post has no usable sent_at (an agent older than 1.12.0)."""
+    if not isinstance(sent_at, str):
+        return None
+    try:
+        sent = _parse_ts(sent_at)
+    except ValueError:
+        return None
+    if sent.tzinfo is None:
+        return None
+    return round((sent - received_at).total_seconds(), 2)
+
+
+def clock_words(offset: float | None, thresholds: dict) -> tuple[str, str]:
+    """('clock 0.3s ahead', 'ok') or ('clock 2m 10s behind', 'warn'): how far
+    a host's clock is from Steward's, graded by Settings → Thresholds' clock
+    drift, either way."""
+    from steward.core.formatting import human_duration, status_class
+    if offset is None:
+        return "", "unknown"
+    off = abs(offset)
+    words = "clock in step" if off < 1 else \
+        f"clock {human_duration(off)} {'ahead' if offset > 0 else 'behind'}"
+    return words, status_class(off, "clock", thresholds)
+
+
 def pick_host_address(current: str | None, reported: str | None) -> str | None:
     """Return the address to store on the Host, or None for no change.
 
@@ -172,6 +201,8 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]
 @host_agent_bp.post("/ingest")
 @public_route("the agent authenticates with its bearer token")
 async def ingest():
+    # Taken first, so the clock offset carries as little of this handler as it can.
+    received_at = datetime.now(timezone.utc)
     auth = request.headers.get("Authorization", "")
     if not auth.startswith("Bearer "):
         return _error(401, "invalid_token")
@@ -261,6 +292,13 @@ async def ingest():
             if accepted == 0:
                 await session.rollback()
                 return _error(400, "malformed_payload", "no valid samples")
+            # How far this host's clock is from Steward's, either way, so an
+            # alert rule reads "> 30" (milestone 490).
+            offset = clock_offset(payload.get("sent_at"), received_at)
+            if offset is not None:
+                metric_rows.append(PluginMetric(
+                    source_module=SOURCE_MODULE, resource_name=host.name,
+                    metric_name="clock_off_secs", value=abs(offset), recorded_at=received_at))
             phases.mark("parse")
 
             # Through the alert pipeline, so rules on host metrics can fire.
@@ -316,11 +354,13 @@ async def ingest():
             if changed:
                 reg.updated_at = datetime.now(timezone.utc)
 
-            if latest_ts:
-                skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
-                if skew > 300:
+            if offset is not None:
+                reg.clock_offset_s, reg.clock_checked_at = offset, received_at
+                clock_th = current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30}
+                if abs(offset) > clock_th["warn"]:
                     current_app.logger.warning(
-                        "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
+                        "host_agent ingest: host=%s clock is %.1fs %s Steward's",
+                        host.name, abs(offset), "ahead of" if offset > 0 else "behind")
 
             await session.commit()
             phases.mark("commit")
@@ -712,6 +752,7 @@ async def host_vitals(host_id: str):
         "_host_vitals.html",
         reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
         sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
+        clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
     )