feat(host_agent): how far each host's clock is from Steward's (milestone 490)

- Agent 1.12.0 stamps each post with sent_at; ingest compares it with the
  moment the post arrived: clock_offset_s / clock_checked_at on the
  registration (host_agent_003), and a clock_off_secs metric (either way)
  that alert rules can watch. Replaces the old skew log, which measured the
  newest sample and was off by up to an interval.
- Settings -> Thresholds: Clock drift, 30 s warn / 5 min critical.
- Host page: 'clock 0.3s ahead' in the vitals line, toned past the warning.
- Hosts list: names hosts whose clock is off; when every reporting host is
  off by about the same amount, says Steward's own clock is the likely one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -36,9 +36,11 @@ async def mcp_overview(db) -> list[dict]:
 def get_alert_metrics() -> dict[str, list[str]]:
     """Host metrics ingest records for alert rules (resource = the host name;
     disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
-    comes from the scheduler, not the agent: "agent offline" is > 180."""
+    comes from the scheduler, not the agent: "agent offline" is > 180.
+    clock_off_secs is how far the host's clock is from Steward's, either way
+    (agent ≥ 1.12.0): "clock drifting" is > 30."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
-        "uptime_secs", "seconds_since_seen",
+        "uptime_secs", "seconds_since_seen", "clock_off_secs",
     ]}

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.11.0"
+AGENT_VERSION = "1.12.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1320,10 +1320,13 @@ def build_sample(mounts: list[str], state: dict,
 
 
 def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
+    # sent_at is this host's clock at the moment of sending: Steward compares
+    # it with its own on arrival to say how far apart the two clocks are.
     return {
         "agent_version": AGENT_VERSION,
         "hostname": hostname,
         "metadata": metadata,
+        "sent_at": datetime.now(timezone.utc).isoformat(),
         "samples": samples,
     }
 

plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py

@@ -0,0 +1,24 @@
+# plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py
+"""host_agent: how far each host's clock is from Steward's (milestone 490)
+
+Revision ID: host_agent_003_clock_offset
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_003_clock_offset"
+down_revision: Union[str, None] = "host_agent_002_host_ip"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_agent_registrations", sa.Column("clock_offset_s", sa.Float(), nullable=True))
+    op.add_column("host_agent_registrations",
+                  sa.Column("clock_checked_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_agent_registrations", "clock_checked_at")
+    op.drop_column("host_agent_registrations", "clock_offset_s")

plugins/host_agent/models.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import Column, String, DateTime, ForeignKey
+from sqlalchemy import Column, String, DateTime, Float, ForeignKey
 from steward.models.base import Base
 
 
@@ -29,6 +29,11 @@ class HostAgentRegistration(Base):
     # Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
     host_ip = Column(String(45), nullable=True)
     last_seen_at = Column(DateTime(timezone=True), nullable=True)
+    # How far this host's clock is from Steward's, in seconds, positive when
+    # the host is ahead, measured from the agent's sent_at on the newest post;
+    # and when that was. None until an agent that stamps its posts reports.
+    clock_offset_s = Column(Float, nullable=True)
+    clock_checked_at = Column(DateTime(timezone=True), nullable=True)
     created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
     updated_at = Column(DateTime(timezone=True), nullable=False,
                         default=_utcnow, onupdate=_utcnow)

plugins/host_agent/routes.py

@@ -44,6 +44,35 @@ def _parse_ts(ts: str) -> datetime:
     return datetime.fromisoformat(ts)
 
 
+def clock_offset(sent_at, received_at: datetime) -> float | None:
+    """Seconds this host's clock is ahead of Steward's (negative: behind),
+    from the agent's sent_at and the moment Steward received the post. It
+    carries the request's own latency, well under a second on a LAN. None
+    when the post has no usable sent_at (an agent older than 1.12.0)."""
+    if not isinstance(sent_at, str):
+        return None
+    try:
+        sent = _parse_ts(sent_at)
+    except ValueError:
+        return None
+    if sent.tzinfo is None:
+        return None
+    return round((sent - received_at).total_seconds(), 2)
+
+
+def clock_words(offset: float | None, thresholds: dict) -> tuple[str, str]:
+    """('clock 0.3s ahead', 'ok') or ('clock 2m 10s behind', 'warn'): how far
+    a host's clock is from Steward's, graded by Settings → Thresholds' clock
+    drift, either way."""
+    from steward.core.formatting import human_duration, status_class
+    if offset is None:
+        return "", "unknown"
+    off = abs(offset)
+    words = "clock in step" if off < 1 else \
+        f"clock {human_duration(off)} {'ahead' if offset > 0 else 'behind'}"
+    return words, status_class(off, "clock", thresholds)
+
+
 def pick_host_address(current: str | None, reported: str | None) -> str | None:
     """Return the address to store on the Host, or None for no change.
 
@@ -172,6 +201,8 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]
 @host_agent_bp.post("/ingest")
 @public_route("the agent authenticates with its bearer token")
 async def ingest():
+    # Taken first, so the clock offset carries as little of this handler as it can.
+    received_at = datetime.now(timezone.utc)
     auth = request.headers.get("Authorization", "")
     if not auth.startswith("Bearer "):
         return _error(401, "invalid_token")
@@ -261,6 +292,13 @@ async def ingest():
             if accepted == 0:
                 await session.rollback()
                 return _error(400, "malformed_payload", "no valid samples")
+            # How far this host's clock is from Steward's, either way, so an
+            # alert rule reads "> 30" (milestone 490).
+            offset = clock_offset(payload.get("sent_at"), received_at)
+            if offset is not None:
+                metric_rows.append(PluginMetric(
+                    source_module=SOURCE_MODULE, resource_name=host.name,
+                    metric_name="clock_off_secs", value=abs(offset), recorded_at=received_at))
             phases.mark("parse")
 
             # Through the alert pipeline, so rules on host metrics can fire.
@@ -316,11 +354,13 @@ async def ingest():
             if changed:
                 reg.updated_at = datetime.now(timezone.utc)
 
-            if latest_ts:
-                skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
-                if skew > 300:
+            if offset is not None:
+                reg.clock_offset_s, reg.clock_checked_at = offset, received_at
+                clock_th = current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30}
+                if abs(offset) > clock_th["warn"]:
                     current_app.logger.warning(
-                        "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
+                        "host_agent ingest: host=%s clock is %.1fs %s Steward's",
+                        host.name, abs(offset), "ahead of" if offset > 0 else "behind")
 
             await session.commit()
             phases.mark("commit")
@@ -712,6 +752,7 @@ async def host_vitals(host_id: str):
         "_host_vitals.html",
         reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
         sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
+        clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
     )
 
 

plugins/host_agent/templates/_host_vitals.html

@@ -16,6 +16,7 @@
     <span class="problem-detail">
       {%- if not stale %}seen {{ reg.last_seen_at|ago }}{% endif %}
       {%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%}
+      {%- if clock and clock[0] %} · <span{% if clock[1] in ('warn', 'crit') %} class="tone-{{ clock[1] }}" title="Further from Steward's clock than Settings → Thresholds allows: check what keeps this host's time in sync"{% endif %}>{{ clock[0] }}</span>{% endif -%}
     </span>
   </div>
   <div class="stat-strip host-health-tiles" style="margin-bottom:0;">