feat(host_agent): how far each host's clock is from Steward's (milestone 490)
- Agent 1.12.0 stamps each post with sent_at; ingest compares it with the moment the post arrived: clock_offset_s / clock_checked_at on the registration (host_agent_003), and a clock_off_secs metric (either way) that alert rules can watch. Replaces the old skew log, which measured the newest sample and was off by up to an interval. - Settings -> Thresholds: Clock drift, 30 s warn / 5 min critical. - Host page: 'clock 0.3s ahead' in the vitals line, toned past the warning. - Hosts list: names hosts whose clock is off; when every reporting host is off by about the same amount, says Steward's own clock is the likely one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
"""The Hosts list calls out a host whose clock is more than 30 s from
|
||||
Steward's, and names Steward's own clock when every host disagrees with it
|
||||
by about the same amount (milestone 490)."""
|
||||
from functools import partial
|
||||
|
||||
from steward.hosts.service import clock_trouble as _trouble
|
||||
|
||||
clock_trouble = partial(_trouble, limit=30) # Settings → Thresholds' default warning
|
||||
|
||||
|
||||
def _a(secs, fresh=True):
|
||||
return {"fresh": fresh, "clock_off_secs": secs}
|
||||
|
||||
|
||||
def test_clocks_in_step_say_nothing():
|
||||
assert clock_trouble({}) is None
|
||||
assert clock_trouble({"a": _a(0.4), "b": _a(29.9), "c": _a(None)}) is None
|
||||
|
||||
|
||||
def test_a_drifting_host_is_named_worst_first():
|
||||
out = clock_trouble({"a": _a(0.2), "b": _a(45), "c": _a(600), "d": _a(900, fresh=False)})
|
||||
# A host that stopped reporting is not judged on an old reading.
|
||||
assert out["hosts"] == [("c", 600), ("b", 45)] and out["steward"] is False
|
||||
|
||||
|
||||
def test_every_host_off_by_the_same_amount_points_at_steward():
|
||||
out = clock_trouble({"a": _a(121), "b": _a(119.5), "c": _a(120.2)})
|
||||
assert out["steward"] is True and out["typical"] == 120.2
|
||||
# One host alone cannot say whose clock is wrong.
|
||||
assert clock_trouble({"a": _a(120)})["steward"] is False
|
||||
# Off by very different amounts: each host is its own problem.
|
||||
assert clock_trouble({"a": _a(40), "b": _a(600)})["steward"] is False
|
||||
# One host in step: Steward's clock agrees with it.
|
||||
assert clock_trouble({"a": _a(120), "b": _a(121), "c": _a(0.3)})["steward"] is False
|
||||
Reference in New Issue
Block a user