fix(mcp): get_anomaly shows a quick Steward restart, not only a gap over 90 s (#5862)
A redeploy's new process ticks at once, so its once-a-minute samples never broke for 90 s and get_anomaly showed none of the day's restarts. The first self-monitor tick in a process now records restart_gap_secs (time since the last process's last sample), and steward_gaps shows each one as a restart. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -144,3 +144,25 @@ def test_unwatched_counts_only_a_real_gap():
|
||||
assert sm.unwatched_secs(None, now) == 0.0 # first ever tick
|
||||
assert sm.unwatched_secs(now - timedelta(seconds=61), now) == 0.0 # an ordinary minute
|
||||
assert sm.unwatched_secs(now - timedelta(minutes=7), now) == 420.0 # a deploy or outage
|
||||
|
||||
|
||||
def test_a_fresh_process_marks_its_restart_once(monkeypatch):
|
||||
"""#5862: the first tick records how long since the last process's last
|
||||
sample, however short; later ticks and a first-ever start don't."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
recorded = _patch_recorder(monkeypatch)
|
||||
monkeypatch.setattr(sm, "_started", False)
|
||||
monkeypatch.setattr(_FakeSession, "last", datetime.now(timezone.utc) - timedelta(seconds=40))
|
||||
|
||||
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
||||
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
||||
|
||||
gaps = [v for (_s, _r, m, v) in recorded if m == "restart_gap_secs"]
|
||||
assert len(gaps) == 1 and 39 <= gaps[0] <= 60
|
||||
assert dict((m, v) for (_s, _r, m, v) in recorded)["unwatched_secs"] == 0.0
|
||||
|
||||
recorded.clear()
|
||||
monkeypatch.setattr(sm, "_started", False)
|
||||
monkeypatch.setattr(_FakeSession, "last", None)
|
||||
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
||||
assert "restart_gap_secs" not in {m for (_s, _r, m, _v) in recorded}
|
||||
|
||||
@@ -137,3 +137,18 @@ def test_steward_not_recording_is_an_event_beside_every_anomaly():
|
||||
assert gap["until"] == "2026-10-15T12:04:10Z" and "wasn't recording" in gap["what"]
|
||||
assert _events.gaps_in(ticks, NOW + timedelta(minutes=5), NOW + timedelta(hours=1)) == []
|
||||
assert _events.gaps_in([NOW + timedelta(minutes=m) for m in range(5)], NOW, NOW + timedelta(hours=1)) == []
|
||||
|
||||
|
||||
def test_a_quick_restart_is_shown_though_it_never_broke_the_minute():
|
||||
"""A redeploy's new process ticks at once, so the samples stay under a
|
||||
minute apart; its first tick's restart_gap_secs marks it (#5862)."""
|
||||
ticks = [NOW, NOW + timedelta(seconds=60), NOW + timedelta(seconds=95), NOW + timedelta(seconds=155)]
|
||||
restart = NOW + timedelta(seconds=95)
|
||||
[ev] = _events.gaps_in(ticks, NOW, NOW + timedelta(hours=1), [(restart, 35.0)])
|
||||
assert ev["kind"] == "steward" and "restarted" in ev["what"]
|
||||
assert (ev["at"], ev["until"]) == ("2026-10-15T12:01:00Z", "2026-10-15T12:01:35Z")
|
||||
# A long restart is one event, not a restart and a gap.
|
||||
slow = NOW + timedelta(minutes=5)
|
||||
[ev] = _events.gaps_in([NOW, slow], NOW, NOW + timedelta(hours=1), [(slow, 300.0)])
|
||||
assert "restarted" in ev["what"]
|
||||
assert _events.gaps_in(ticks, NOW + timedelta(minutes=2), NOW + timedelta(hours=1), [(restart, 35.0)]) == []
|
||||
|
||||
Reference in New Issue
Block a user