From 96c1ea94077f7b3cf19933a2cb41a1538dc065db Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 8 Oct 2026 08:00:33 -0400 Subject: [PATCH] feat(host_agent): how far each host's clock is from Steward's (milestone 490) - Agent 1.12.0 stamps each post with sent_at; ingest compares it with the moment the post arrived: clock_offset_s / clock_checked_at on the registration (host_agent_003), and a clock_off_secs metric (either way) that alert rules can watch. Replaces the old skew log, which measured the newest sample and was off by up to an interval. - Settings -> Thresholds: Clock drift, 30 s warn / 5 min critical. - Host page: 'clock 0.3s ahead' in the vitals line, toned past the warning. - Hosts list: names hosts whose clock is off; when every reporting host is off by about the same amount, says Steward's own clock is the likely one. Co-Authored-By: Claude Opus 5.5 --- plugins/host_agent/__init__.py | 6 +- plugins/host_agent/agent.py | 5 +- .../versions/host_agent_003_clock_offset.py | 24 ++++++++ plugins/host_agent/models.py | 7 ++- plugins/host_agent/routes.py | 49 ++++++++++++++-- .../host_agent/templates/_host_vitals.html | 1 + steward/core/formatting.py | 1 + steward/core/settings.py | 4 ++ steward/hosts/routes.py | 5 +- steward/hosts/service.py | 25 +++++++- steward/settings/routes.py | 1 + steward/templates/hosts/list.html | 8 +++ steward/templates/settings/thresholds.html | 1 + tests/core/test_clock_trouble.py | 34 +++++++++++ tests/plugins/host_agent/test_clock_offset.py | 57 +++++++++++++++++++ 15 files changed, 217 insertions(+), 11 deletions(-) create mode 100644 plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py create mode 100644 tests/core/test_clock_trouble.py create mode 100644 tests/plugins/host_agent/test_clock_offset.py diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index 5b577b3..b579491 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -36,9 +36,11 @@ async def mcp_overview(db) -> list[dict]: def get_alert_metrics() -> dict[str, list[str]]: """Host metrics ingest records for alert rules (resource = the host name; disk_used_pct per mount, resource ":"). seconds_since_seen - comes from the scheduler, not the agent: "agent offline" is > 180.""" + comes from the scheduler, not the agent: "agent offline" is > 180. + clock_off_secs is how far the host's clock is from Steward's, either way + (agent ≥ 1.12.0): "clock drifting" is > 30.""" return {"host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", - "uptime_secs", "seconds_since_seen", + "uptime_secs", "seconds_since_seen", "clock_off_secs", ]} diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 493e14c..6e5160c 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -20,7 +20,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor, wait as wait_futures from datetime import datetime, timezone -AGENT_VERSION = "1.11.0" +AGENT_VERSION = "1.12.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -1320,10 +1320,13 @@ def build_sample(mounts: list[str], state: dict, def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict: + # sent_at is this host's clock at the moment of sending: Steward compares + # it with its own on arrival to say how far apart the two clocks are. return { "agent_version": AGENT_VERSION, "hostname": hostname, "metadata": metadata, + "sent_at": datetime.now(timezone.utc).isoformat(), "samples": samples, } diff --git a/plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py b/plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py new file mode 100644 index 0000000..16f7163 --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py @@ -0,0 +1,24 @@ +# plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py +"""host_agent: how far each host's clock is from Steward's (milestone 490) + +Revision ID: host_agent_003_clock_offset +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_003_clock_offset" +down_revision: Union[str, None] = "host_agent_002_host_ip" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("host_agent_registrations", sa.Column("clock_offset_s", sa.Float(), nullable=True)) + op.add_column("host_agent_registrations", + sa.Column("clock_checked_at", sa.DateTime(timezone=True), nullable=True)) + + +def downgrade() -> None: + op.drop_column("host_agent_registrations", "clock_checked_at") + op.drop_column("host_agent_registrations", "clock_offset_s") diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py index e8d87d0..cc61f09 100644 --- a/plugins/host_agent/models.py +++ b/plugins/host_agent/models.py @@ -2,7 +2,7 @@ from __future__ import annotations import uuid from datetime import datetime, timezone -from sqlalchemy import Column, String, DateTime, ForeignKey +from sqlalchemy import Column, String, DateTime, Float, ForeignKey from steward.models.base import Base @@ -29,6 +29,11 @@ class HostAgentRegistration(Base): # Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix). host_ip = Column(String(45), nullable=True) last_seen_at = Column(DateTime(timezone=True), nullable=True) + # How far this host's clock is from Steward's, in seconds, positive when + # the host is ahead, measured from the agent's sent_at on the newest post; + # and when that was. None until an agent that stamps its posts reports. + clock_offset_s = Column(Float, nullable=True) + clock_checked_at = Column(DateTime(timezone=True), nullable=True) created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) updated_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow, onupdate=_utcnow) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 03be356..9aa4ff0 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -44,6 +44,35 @@ def _parse_ts(ts: str) -> datetime: return datetime.fromisoformat(ts) +def clock_offset(sent_at, received_at: datetime) -> float | None: + """Seconds this host's clock is ahead of Steward's (negative: behind), + from the agent's sent_at and the moment Steward received the post. It + carries the request's own latency, well under a second on a LAN. None + when the post has no usable sent_at (an agent older than 1.12.0).""" + if not isinstance(sent_at, str): + return None + try: + sent = _parse_ts(sent_at) + except ValueError: + return None + if sent.tzinfo is None: + return None + return round((sent - received_at).total_seconds(), 2) + + +def clock_words(offset: float | None, thresholds: dict) -> tuple[str, str]: + """('clock 0.3s ahead', 'ok') or ('clock 2m 10s behind', 'warn'): how far + a host's clock is from Steward's, graded by Settings → Thresholds' clock + drift, either way.""" + from steward.core.formatting import human_duration, status_class + if offset is None: + return "", "unknown" + off = abs(offset) + words = "clock in step" if off < 1 else \ + f"clock {human_duration(off)} {'ahead' if offset > 0 else 'behind'}" + return words, status_class(off, "clock", thresholds) + + def pick_host_address(current: str | None, reported: str | None) -> str | None: """Return the address to store on the Host, or None for no change. @@ -172,6 +201,8 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int] @host_agent_bp.post("/ingest") @public_route("the agent authenticates with its bearer token") async def ingest(): + # Taken first, so the clock offset carries as little of this handler as it can. + received_at = datetime.now(timezone.utc) auth = request.headers.get("Authorization", "") if not auth.startswith("Bearer "): return _error(401, "invalid_token") @@ -261,6 +292,13 @@ async def ingest(): if accepted == 0: await session.rollback() return _error(400, "malformed_payload", "no valid samples") + # How far this host's clock is from Steward's, either way, so an + # alert rule reads "> 30" (milestone 490). + offset = clock_offset(payload.get("sent_at"), received_at) + if offset is not None: + metric_rows.append(PluginMetric( + source_module=SOURCE_MODULE, resource_name=host.name, + metric_name="clock_off_secs", value=abs(offset), recorded_at=received_at)) phases.mark("parse") # Through the alert pipeline, so rules on host metrics can fire. @@ -316,11 +354,13 @@ async def ingest(): if changed: reg.updated_at = datetime.now(timezone.utc) - if latest_ts: - skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds()) - if skew > 300: + if offset is not None: + reg.clock_offset_s, reg.clock_checked_at = offset, received_at + clock_th = current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30} + if abs(offset) > clock_th["warn"]: current_app.logger.warning( - "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name) + "host_agent ingest: host=%s clock is %.1fs %s Steward's", + host.name, abs(offset), "ahead of" if offset > 0 else "behind") await session.commit() phases.mark("commit") @@ -712,6 +752,7 @@ async def host_vitals(host_id: str): "_host_vitals.html", reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core, sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst, + clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})), ) diff --git a/plugins/host_agent/templates/_host_vitals.html b/plugins/host_agent/templates/_host_vitals.html index 0025a71..843cb09 100644 --- a/plugins/host_agent/templates/_host_vitals.html +++ b/plugins/host_agent/templates/_host_vitals.html @@ -16,6 +16,7 @@ {%- if not stale %}seen {{ reg.last_seen_at|ago }}{% endif %} {%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%} + {%- if clock and clock[0] %} · {{ clock[0] }}{% endif -%}
diff --git a/steward/core/formatting.py b/steward/core/formatting.py index d5738a0..aefa87b 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -285,6 +285,7 @@ METRICS: dict[str, tuple[str, str]] = { "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"), "uptime_secs": ("Uptime", "time since the host booted"), "seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"), + "clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"), } # Where a rule's metric comes from, as the picker and the rules list say it. diff --git a/steward/core/settings.py b/steward/core/settings.py index 3c70c12..75f4098 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -97,6 +97,9 @@ DEFAULTS: dict[str, Any] = { # many container deaths within an hour count as crash-looping. "thresholds.tls_warn_days": 14, "thresholds.tls_crit_days": 3, "thresholds.crash_loop_deaths": 3, + # Seconds a host's clock may be from Steward's before it is called out + # (milestone 490), either way. + "thresholds.clock_warn_s": 30, "thresholds.clock_crit_s": 300, # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s # samples are heavy, so keep a short raw window then roll up to hourly # averages kept much longer; lifecycle events are light, keep a month. @@ -448,6 +451,7 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict: "latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"}, "uptime": {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"}, "tls_days": {"warn": g("thresholds.tls_warn_days", 14), "crit": g("thresholds.tls_crit_days", 3), "dir": "low"}, + "clock": {"warn": g("thresholds.clock_warn_s", 30), "crit": g("thresholds.clock_crit_s", 300), "dir": "high"}, "crash_loop": {"warn": g("thresholds.crash_loop_deaths", 3), "crit": g("thresholds.crash_loop_deaths", 3), "dir": "high"}, } diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index 49677f1..a310cd7 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -92,7 +92,10 @@ async def _list_page(add: dict | None = None): ctx = dict(hosts=shown, total=len(hosts), host_status=host_status, uptime=uptime, agent=agent, groups=groups, group_names=group_names, f=f, filter_url=filter_url, sort_url=sort_url, - states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add) + states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add, + clock=host_svc.clock_trouble( + agent, (current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30})["warn"]), + host_ids={h.name: h.id for h in hosts}) # The filter bar's swap gets the table alone. A history restore (Back to a # pushed URL) is an HTMX request too but wants the page, so key on the # target; Vary keeps the browser from caching one as the other. diff --git a/steward/hosts/service.py b/steward/hosts/service.py index 9240838..53c2bbf 100644 --- a/steward/hosts/service.py +++ b/steward/hosts/service.py @@ -249,7 +249,7 @@ async def agent_overview_by_host( """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest. {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m, - cores, load_per_core, last_at, fresh}} for each host with any + cores, load_per_core, last_at, fresh, clock_off_secs}} for each host with any agent samples. Cores are counted from the per-core CPU sub-resources (":core"), as on the host page; load_per_core is load_1m as a % of them, the figure Settings → Thresholds grades "load" by. @@ -266,7 +266,7 @@ async def agent_overview_by_host( stale_after = int(stale_after_seconds) latest = await latest_by_resource( db, "host_agent", list(host_names) + [h + ":/" for h in host_names], - set(AGENT_GLANCE_METRICS) | {"disk_used_pct"}) + set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"}) from steward.models.metrics import PluginMetricLatest cores: dict[str, int] = {} for res in (await db.execute( @@ -289,6 +289,8 @@ async def agent_overview_by_host( ts = max((r.recorded_at for r in own.values()), default=None) d["last_at"] = ts d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after) + clock = latest.get(name, {}).get("clock_off_secs") + d["clock_off_secs"] = clock.value if clock is not None else None d["cores"] = cores.get(name) load = d.get("load_1m") d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None @@ -296,6 +298,25 @@ async def agent_overview_by_host( return out +def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None: + """Reporting hosts whose clock is more than `limit` seconds from Steward's + (Settings → Thresholds' clock drift warning, milestone 490), + worst first: {"hosts": [(name, seconds)], "steward": bool, "typical": s}. + + `steward` is set when every reporting host (two or more) is off, and by + about the same amount: then the one clock they all disagree with, + Steward's own, is the likely culprit. None when every clock is in step.""" + readings = {n: a["clock_off_secs"] for n, a in agent.items() + if a.get("fresh") and a.get("clock_off_secs") is not None} + off = {n: v for n, v in readings.items() if v > limit} + if not off: + return None + worst = sorted(off.items(), key=lambda kv: (-kv[1], kv[0])) + low, high = min(off.values()), max(off.values()) + together = len(readings) >= 2 and len(off) == len(readings) and high - low <= max(5.0, 0.2 * low) + return {"hosts": worst, "steward": together, "typical": sorted(off.values())[len(off) // 2]} + + GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5} diff --git a/steward/settings/routes.py b/steward/settings/routes.py index bacd81a..833f6b6 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -138,6 +138,7 @@ _THRESHOLD_FIELDS = [ ("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True), ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False), ("tls_warn", "thresholds.tls_warn_days", False), ("tls_crit", "thresholds.tls_crit_days", False), + ("clock_warn", "thresholds.clock_warn_s", False), ("clock_crit", "thresholds.clock_crit_s", False), ("crash_loop_deaths", "thresholds.crash_loop_deaths", False), ] diff --git a/steward/templates/hosts/list.html b/steward/templates/hosts/list.html index 25d5b9a..c8211e1 100644 --- a/steward/templates/hosts/list.html +++ b/steward/templates/hosts/list.html @@ -22,6 +22,14 @@ {% endif %}
+{% if clock %} +{#- Milestone 490: a host whose clock drifts, or Steward's own when every host disagrees with it. #} +

+ {%- if clock.steward %}Every host's clock is about {{ clock.typical|duration }} off from Steward's: Steward's own clock is the likely one that is wrong. Check what keeps the time in sync on the machine Steward runs on. + {%- else %}Clock off from Steward's on {% for name, secs in clock.hosts %}{{ name }} by {{ secs|duration }}{{ ", " if not loop.last }}{% endfor %}. Check what keeps the time in sync there. + {%- endif -%} +

+{% endif %} {% if total %}