feat(host_agent): how far each host's clock is from Steward's (milestone 490)

- Agent 1.12.0 stamps each post with sent_at; ingest compares it with the
  moment the post arrived: clock_offset_s / clock_checked_at on the
  registration (host_agent_003), and a clock_off_secs metric (either way)
  that alert rules can watch. Replaces the old skew log, which measured the
  newest sample and was off by up to an interval.
- Settings -> Thresholds: Clock drift, 30 s warn / 5 min critical.
- Host page: 'clock 0.3s ahead' in the vitals line, toned past the warning.
- Hosts list: names hosts whose clock is off; when every reporting host is
  off by about the same amount, says Steward's own clock is the likely one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -36,9 +36,11 @@ async def mcp_overview(db) -> list[dict]:
 def get_alert_metrics() -> dict[str, list[str]]:
     """Host metrics ingest records for alert rules (resource = the host name;
     disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
-    comes from the scheduler, not the agent: "agent offline" is > 180."""
+    comes from the scheduler, not the agent: "agent offline" is > 180.
+    clock_off_secs is how far the host's clock is from Steward's, either way
+    (agent ≥ 1.12.0): "clock drifting" is > 30."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
-        "uptime_secs", "seconds_since_seen",
+        "uptime_secs", "seconds_since_seen", "clock_off_secs",
     ]}

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.11.0"
+AGENT_VERSION = "1.12.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1320,10 +1320,13 @@ def build_sample(mounts: list[str], state: dict,
 
 
 def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
+    # sent_at is this host's clock at the moment of sending: Steward compares
+    # it with its own on arrival to say how far apart the two clocks are.
     return {
         "agent_version": AGENT_VERSION,
         "hostname": hostname,
         "metadata": metadata,
+        "sent_at": datetime.now(timezone.utc).isoformat(),
         "samples": samples,
     }
 

plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py

@@ -0,0 +1,24 @@
+# plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py
+"""host_agent: how far each host's clock is from Steward's (milestone 490)
+
+Revision ID: host_agent_003_clock_offset
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_003_clock_offset"
+down_revision: Union[str, None] = "host_agent_002_host_ip"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_agent_registrations", sa.Column("clock_offset_s", sa.Float(), nullable=True))
+    op.add_column("host_agent_registrations",
+                  sa.Column("clock_checked_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_agent_registrations", "clock_checked_at")
+    op.drop_column("host_agent_registrations", "clock_offset_s")

plugins/host_agent/models.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import Column, String, DateTime, ForeignKey
+from sqlalchemy import Column, String, DateTime, Float, ForeignKey
 from steward.models.base import Base
 
 
@@ -29,6 +29,11 @@ class HostAgentRegistration(Base):
     # Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
     host_ip = Column(String(45), nullable=True)
     last_seen_at = Column(DateTime(timezone=True), nullable=True)
+    # How far this host's clock is from Steward's, in seconds, positive when
+    # the host is ahead, measured from the agent's sent_at on the newest post;
+    # and when that was. None until an agent that stamps its posts reports.
+    clock_offset_s = Column(Float, nullable=True)
+    clock_checked_at = Column(DateTime(timezone=True), nullable=True)
     created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
     updated_at = Column(DateTime(timezone=True), nullable=False,
                         default=_utcnow, onupdate=_utcnow)

plugins/host_agent/routes.py

@@ -44,6 +44,35 @@ def _parse_ts(ts: str) -> datetime:
     return datetime.fromisoformat(ts)
 
 
+def clock_offset(sent_at, received_at: datetime) -> float | None:
+    """Seconds this host's clock is ahead of Steward's (negative: behind),
+    from the agent's sent_at and the moment Steward received the post. It
+    carries the request's own latency, well under a second on a LAN. None
+    when the post has no usable sent_at (an agent older than 1.12.0)."""
+    if not isinstance(sent_at, str):
+        return None
+    try:
+        sent = _parse_ts(sent_at)
+    except ValueError:
+        return None
+    if sent.tzinfo is None:
+        return None
+    return round((sent - received_at).total_seconds(), 2)
+
+
+def clock_words(offset: float | None, thresholds: dict) -> tuple[str, str]:
+    """('clock 0.3s ahead', 'ok') or ('clock 2m 10s behind', 'warn'): how far
+    a host's clock is from Steward's, graded by Settings → Thresholds' clock
+    drift, either way."""
+    from steward.core.formatting import human_duration, status_class
+    if offset is None:
+        return "", "unknown"
+    off = abs(offset)
+    words = "clock in step" if off < 1 else \
+        f"clock {human_duration(off)} {'ahead' if offset > 0 else 'behind'}"
+    return words, status_class(off, "clock", thresholds)
+
+
 def pick_host_address(current: str | None, reported: str | None) -> str | None:
     """Return the address to store on the Host, or None for no change.
 
@@ -172,6 +201,8 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]
 @host_agent_bp.post("/ingest")
 @public_route("the agent authenticates with its bearer token")
 async def ingest():
+    # Taken first, so the clock offset carries as little of this handler as it can.
+    received_at = datetime.now(timezone.utc)
     auth = request.headers.get("Authorization", "")
     if not auth.startswith("Bearer "):
         return _error(401, "invalid_token")
@@ -261,6 +292,13 @@ async def ingest():
             if accepted == 0:
                 await session.rollback()
                 return _error(400, "malformed_payload", "no valid samples")
+            # How far this host's clock is from Steward's, either way, so an
+            # alert rule reads "> 30" (milestone 490).
+            offset = clock_offset(payload.get("sent_at"), received_at)
+            if offset is not None:
+                metric_rows.append(PluginMetric(
+                    source_module=SOURCE_MODULE, resource_name=host.name,
+                    metric_name="clock_off_secs", value=abs(offset), recorded_at=received_at))
             phases.mark("parse")
 
             # Through the alert pipeline, so rules on host metrics can fire.
@@ -316,11 +354,13 @@ async def ingest():
             if changed:
                 reg.updated_at = datetime.now(timezone.utc)
 
-            if latest_ts:
+            if offset is not None:
-                skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
+                reg.clock_offset_s, reg.clock_checked_at = offset, received_at
-                if skew > 300:
+                clock_th = current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30}
+                if abs(offset) > clock_th["warn"]:
                     current_app.logger.warning(
-                        "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
+                        "host_agent ingest: host=%s clock is %.1fs %s Steward's",
+                        host.name, abs(offset), "ahead of" if offset > 0 else "behind")
 
             await session.commit()
             phases.mark("commit")
@@ -712,6 +752,7 @@ async def host_vitals(host_id: str):
         "_host_vitals.html",
         reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
         sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
+        clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
     )
 
 

plugins/host_agent/templates/_host_vitals.html

@@ -16,6 +16,7 @@
     <span class="problem-detail">
       {%- if not stale %}seen {{ reg.last_seen_at|ago }}{% endif %}
       {%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%}
+      {%- if clock and clock[0] %} · <span{% if clock[1] in ('warn', 'crit') %} class="tone-{{ clock[1] }}" title="Further from Steward's clock than Settings → Thresholds allows: check what keeps this host's time in sync"{% endif %}>{{ clock[0] }}</span>{% endif -%}
     </span>
   </div>
   <div class="stat-strip host-health-tiles" style="margin-bottom:0;">

steward/core/formatting.py

@@ -285,6 +285,7 @@ METRICS: dict[str, tuple[str, str]] = {
     "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"),
     "uptime_secs": ("Uptime", "time since the host booted"),
     "seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"),
+    "clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"),
 }
 
 # Where a rule's metric comes from, as the picker and the rules list say it.

steward/core/settings.py

@@ -97,6 +97,9 @@ DEFAULTS: dict[str, Any] = {
     # many container deaths within an hour count as crash-looping.
     "thresholds.tls_warn_days": 14, "thresholds.tls_crit_days": 3,
     "thresholds.crash_loop_deaths": 3,
+    # Seconds a host's clock may be from Steward's before it is called out
+    # (milestone 490), either way.
+    "thresholds.clock_warn_s": 30, "thresholds.clock_crit_s": 300,
     # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s
     # samples are heavy, so keep a short raw window then roll up to hourly
     # averages kept much longer; lifecycle events are light, keep a month.
@@ -448,6 +451,7 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
         "latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"},
         "uptime":  {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"},
         "tls_days": {"warn": g("thresholds.tls_warn_days", 14), "crit": g("thresholds.tls_crit_days", 3), "dir": "low"},
+        "clock":   {"warn": g("thresholds.clock_warn_s", 30), "crit": g("thresholds.clock_crit_s", 300), "dir": "high"},
         "crash_loop": {"warn": g("thresholds.crash_loop_deaths", 3), "crit": g("thresholds.crash_loop_deaths", 3), "dir": "high"},
     }
 

steward/hosts/routes.py

@@ -92,7 +92,10 @@ async def _list_page(add: dict | None = None):
 
     ctx = dict(hosts=shown, total=len(hosts), host_status=host_status, uptime=uptime, agent=agent,
                groups=groups, group_names=group_names, f=f, filter_url=filter_url, sort_url=sort_url,
-               states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add)
+               states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add,
+               clock=host_svc.clock_trouble(
+                   agent, (current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30})["warn"]),
+               host_ids={h.name: h.id for h in hosts})
     # The filter bar's swap gets the table alone. A history restore (Back to a
     # pushed URL) is an HTMX request too but wants the page, so key on the
     # target; Vary keeps the browser from caching one as the other.

steward/hosts/service.py

@@ -249,7 +249,7 @@ async def agent_overview_by_host(
     """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
 
     {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
-                 cores, load_per_core, last_at, fresh}} for each host with any
+                 cores, load_per_core, last_at, fresh, clock_off_secs}} for each host with any
     agent samples. Cores are counted from the per-core CPU sub-resources
     ("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
     of them, the figure Settings → Thresholds grades "load" by.
@@ -266,7 +266,7 @@ async def agent_overview_by_host(
     stale_after = int(stale_after_seconds)
     latest = await latest_by_resource(
         db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
-        set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
+        set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"})
     from steward.models.metrics import PluginMetricLatest
     cores: dict[str, int] = {}
     for res in (await db.execute(
@@ -289,6 +289,8 @@ async def agent_overview_by_host(
         ts = max((r.recorded_at for r in own.values()), default=None)
         d["last_at"] = ts
         d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
+        clock = latest.get(name, {}).get("clock_off_secs")
+        d["clock_off_secs"] = clock.value if clock is not None else None
         d["cores"] = cores.get(name)
         load = d.get("load_1m")
         d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
@@ -296,6 +298,25 @@ async def agent_overview_by_host(
     return out
 
 
+def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None:
+    """Reporting hosts whose clock is more than `limit` seconds from Steward's
+    (Settings → Thresholds' clock drift warning, milestone 490),
+    worst first: {"hosts": [(name, seconds)], "steward": bool, "typical": s}.
+
+    `steward` is set when every reporting host (two or more) is off, and by
+    about the same amount: then the one clock they all disagree with,
+    Steward's own, is the likely culprit. None when every clock is in step."""
+    readings = {n: a["clock_off_secs"] for n, a in agent.items()
+                if a.get("fresh") and a.get("clock_off_secs") is not None}
+    off = {n: v for n, v in readings.items() if v > limit}
+    if not off:
+        return None
+    worst = sorted(off.items(), key=lambda kv: (-kv[1], kv[0]))
+    low, high = min(off.values()), max(off.values())
+    together = len(readings) >= 2 and len(off) == len(readings) and high - low <= max(5.0, 0.2 * low)
+    return {"hosts": worst, "steward": together, "typical": sorted(off.values())[len(off) // 2]}
+
+
 GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5}
 
 

steward/settings/routes.py

@@ -138,6 +138,7 @@ _THRESHOLD_FIELDS = [
     ("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True),
     ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False),
     ("tls_warn", "thresholds.tls_warn_days", False), ("tls_crit", "thresholds.tls_crit_days", False),
+    ("clock_warn", "thresholds.clock_warn_s", False), ("clock_crit", "thresholds.clock_crit_s", False),
     ("crash_loop_deaths", "thresholds.crash_loop_deaths", False),
 ]
 

steward/templates/hosts/list.html

@@ -22,6 +22,14 @@
         {% endif %}
     </div>
 </div>
+{% if clock %}
+{#- Milestone 490: a host whose clock drifts, or Steward's own when every host disagrees with it. #}
+<p class="page-verdict tone-warn">
+  {%- if clock.steward %}Every host's clock is about {{ clock.typical|duration }} off from Steward's: Steward's own clock is the likely one that is wrong. Check what keeps the time in sync on the machine Steward runs on.
+  {%- else %}Clock off from Steward's on {% for name, secs in clock.hosts %}<a href="/hosts/{{ host_ids[name] }}">{{ name }}</a> by {{ secs|duration }}{{ ", " if not loop.last }}{% endfor %}. Check what keeps the time in sync there.
+  {%- endif -%}
+</p>
+{% endif %}
 {% if total %}
 <form class="filter-bar" method="get" action="/hosts/" role="search"
       hx-target="#hosts-table" hx-push-url="true" hx-include="closest form">

steward/templates/settings/thresholds.html

@@ -43,6 +43,7 @@
   {{ pair("Ping latency", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "Round-trip time. Shared with the Ping widget's good/warn colouring.") }}
   {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
   {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }}
+  {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }}
 
   <div class="form-group" style="margin-bottom:0.25rem;">
     <label>Container crash loop <span style="color:var(--text-muted);font-size:0.8rem;">(deaths per hour)</span></label>

tests/core/test_clock_trouble.py

@@ -0,0 +1,34 @@
+"""The Hosts list calls out a host whose clock is more than 30 s from
+Steward's, and names Steward's own clock when every host disagrees with it
+by about the same amount (milestone 490)."""
+from functools import partial
+
+from steward.hosts.service import clock_trouble as _trouble
+
+clock_trouble = partial(_trouble, limit=30)   # Settings → Thresholds' default warning
+
+
+def _a(secs, fresh=True):
+    return {"fresh": fresh, "clock_off_secs": secs}
+
+
+def test_clocks_in_step_say_nothing():
+    assert clock_trouble({}) is None
+    assert clock_trouble({"a": _a(0.4), "b": _a(29.9), "c": _a(None)}) is None
+
+
+def test_a_drifting_host_is_named_worst_first():
+    out = clock_trouble({"a": _a(0.2), "b": _a(45), "c": _a(600), "d": _a(900, fresh=False)})
+    # A host that stopped reporting is not judged on an old reading.
+    assert out["hosts"] == [("c", 600), ("b", 45)] and out["steward"] is False
+
+
+def test_every_host_off_by_the_same_amount_points_at_steward():
+    out = clock_trouble({"a": _a(121), "b": _a(119.5), "c": _a(120.2)})
+    assert out["steward"] is True and out["typical"] == 120.2
+    # One host alone cannot say whose clock is wrong.
+    assert clock_trouble({"a": _a(120)})["steward"] is False
+    # Off by very different amounts: each host is its own problem.
+    assert clock_trouble({"a": _a(40), "b": _a(600)})["steward"] is False
+    # One host in step: Steward's clock agrees with it.
+    assert clock_trouble({"a": _a(120), "b": _a(121), "c": _a(0.3)})["steward"] is False

tests/plugins/host_agent/test_clock_offset.py

@@ -0,0 +1,57 @@
+"""How far a host's clock is from Steward's (milestone 490): the agent stamps
+each post with sent_at, Steward compares it with the moment it arrived, and
+the host page says it in words, graded by Settings → Thresholds' clock
+drift (30 s warn, 5 min critical by default)."""
+from datetime import datetime, timezone
+
+import pytest
+
+from plugins.host_agent import agent as a
+from plugins.host_agent.routes import clock_offset, clock_words
+from steward.core.settings import to_thresholds_cfg
+
+RECEIVED = datetime(2026, 10, 8, 10, 0, 0, tzinfo=timezone.utc)
+TH = to_thresholds_cfg({})
+
+
+def test_the_agent_stamps_each_post_with_its_own_clock():
+    payload = a.build_payload(samples=[{"ts": "2026-10-08T10:00:00+00:00"}], hostname="h", metadata={})
+    sent = datetime.fromisoformat(payload["sent_at"])
+    assert sent.tzinfo is not None
+    assert abs((sent - datetime.now(timezone.utc)).total_seconds()) < 5
+    assert a.AGENT_VERSION >= "1.12.0"
+
+
+@pytest.mark.parametrize("sent_at, offset", [
+    ("2026-10-08T10:00:05+00:00", 5.0),          # the host is ahead
+    ("2026-10-08T09:57:50Z", -130.0),            # behind, Z suffix
+    ("2026-10-08T06:00:00.250-04:00", 0.25),     # another zone, the same moment
+])
+def test_the_offset_is_the_hosts_clock_against_steward_on_arrival(sent_at, offset):
+    assert clock_offset(sent_at, RECEIVED) == offset
+
+
+@pytest.mark.parametrize("sent_at", [None, "", "yesterday", 1728381600, "2026-10-08T10:00:05"])
+def test_no_usable_stamp_is_no_offset(sent_at):
+    # An agent before 1.12.0 sends none; a naive time says nothing about a zone.
+    assert clock_offset(sent_at, RECEIVED) is None
+
+
+@pytest.mark.parametrize("offset, words, tone", [
+    (None, "", "unknown"),
+    (0.3, "clock in step", "ok"),
+    (-0.9, "clock in step", "ok"),
+    (12.4, "clock 12s ahead", "ok"),
+    (29.9, "clock 29s ahead", "ok"),
+    (-130, "clock 2m 10s behind", "warn"),
+    (31, "clock 31s ahead", "warn"),
+    (-3600, "clock 1h behind", "crit"),
+])
+def test_the_host_page_says_it_in_words(offset, words, tone):
+    assert clock_words(offset, TH) == (words, tone)
+
+
+def test_the_cutoff_is_the_operators():
+    th = to_thresholds_cfg({"thresholds.clock_warn_s": 5, "thresholds.clock_crit_s": 60})
+    assert clock_words(12.4, th) == ("clock 12s ahead", "warn")
+    assert clock_words(-90, th)[1] == "crit"