fix(mcp): get_anomaly shows a quick Steward restart, not only a gap over 90 s (#5862)
A redeploy's new process ticks at once, so its once-a-minute samples never broke for 90 s and get_anomaly showed none of the day's restarts. The first self-monitor tick in a process now records restart_gap_secs (time since the last process's last sample), and steward_gaps shows each one as a restart. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -152,13 +152,23 @@ async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
|
||||
return here, elsewhere
|
||||
|
||||
|
||||
def gaps_in(times: list[datetime], since: datetime, until: datetime) -> list[dict]:
|
||||
"""Each break longer than STEWARD_GAP_S between consecutive samples that
|
||||
overlaps the window, as a "steward" event lasting the break."""
|
||||
def gaps_in(times: list[datetime], since: datetime, until: datetime,
|
||||
restarts: list[tuple[datetime, float]] = ()) -> list[dict]:
|
||||
"""Each restart (its first sample's time and how long since the last
|
||||
one) and each other break longer than STEWARD_GAP_S between consecutive
|
||||
samples that overlaps the window, as a "steward" event lasting the break.
|
||||
A restart is shown however short: a quick redeploy never breaks the
|
||||
minute, and still restarts every count Steward keeps."""
|
||||
out = []
|
||||
for at, secs in restarts:
|
||||
if at >= since and at - timedelta(seconds=secs) <= until:
|
||||
out.append(event(at - timedelta(seconds=secs), "steward",
|
||||
f"Steward restarted, not recording for {human_duration(secs)}: "
|
||||
"readings either side of it can be wrong", until=at))
|
||||
restarted = {at for at, _ in restarts}
|
||||
for before, after in zip(times, times[1:]):
|
||||
secs = (after - before).total_seconds()
|
||||
if secs > STEWARD_GAP_S and after >= since and before <= until:
|
||||
if secs > STEWARD_GAP_S and after >= since and before <= until and after not in restarted:
|
||||
out.append(event(before, "steward",
|
||||
f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a "
|
||||
"stall): readings either side of a gap can be wrong", until=after))
|
||||
@@ -172,12 +182,13 @@ async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]:
|
||||
silence (#5862). Raw samples only, so a window older than their
|
||||
retention shows none."""
|
||||
rows = (await db.execute(
|
||||
select(PluginMetric.recorded_at).where(
|
||||
select(PluginMetric.metric_name, PluginMetric.recorded_at, PluginMetric.value).where(
|
||||
PluginMetric.source_module == "steward", PluginMetric.resource_name == "process",
|
||||
PluginMetric.metric_name == "db_roundtrip_ms",
|
||||
PluginMetric.metric_name.in_(("db_roundtrip_ms", "restart_gap_secs")),
|
||||
PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until)
|
||||
.order_by(PluginMetric.recorded_at))).scalars()
|
||||
return gaps_in(list(rows), since, until)
|
||||
.order_by(PluginMetric.recorded_at))).all()
|
||||
return gaps_in([at for name, at, _ in rows if name == "db_roundtrip_ms"], since, until,
|
||||
[(at, value) for name, at, value in rows if name == "restart_gap_secs"])
|
||||
|
||||
|
||||
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]:
|
||||
|
||||
Reference in New Issue
Block a user