fix(mcp): get_anomaly shows a quick Steward restart, not only a gap over 90 s (#5862)
A redeploy's new process ticks at once, so its once-a-minute samples never broke for 90 s and get_anomaly showed none of the day's restarts. The first self-monitor tick in a process now records restart_gap_secs (time since the last process's last sample), and steward_gaps shows each one as a restart. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,10 @@ source and resource:
|
||||
• ``db_roundtrip_ms`` — a timed ``SELECT 1``, connection checkout included
|
||||
• ``unwatched_secs`` — how long Steward wasn't recording before this tick (a
|
||||
deploy, an outage, a stalled loop), from its own last sample; 0 when unbroken
|
||||
• ``restart_gap_secs`` — once, on a fresh process's first tick: how long since
|
||||
the last process's last sample, however short. A quick redeploy never
|
||||
breaks the minute long enough to show as unwatched; this marks it anyway,
|
||||
so get_anomaly can show Steward's own restarts beside an anomaly (#5862)
|
||||
|
||||
``cleanup_secs`` is recorded by the hourly cleanup itself when it finishes.
|
||||
"""
|
||||
@@ -51,6 +55,7 @@ except ImportError: # pragma: no cover - non-POSIX
|
||||
FD_WARN_PCT = 80.0
|
||||
|
||||
_warned = False # de-dupe the WARNING so a sustained leak doesn't spam the log
|
||||
_started = False # this process has ticked: restart_gap_secs is recorded on the first tick only
|
||||
|
||||
|
||||
def count_open_fds() -> int | None:
|
||||
@@ -98,7 +103,7 @@ async def record_self_metrics(app) -> None:
|
||||
The open-file pair is skipped (logged at debug) where fd accounting isn't
|
||||
available, so it's safe to schedule unconditionally.
|
||||
"""
|
||||
global _warned
|
||||
global _warned, _started
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
@@ -129,11 +134,13 @@ async def record_self_metrics(app) -> None:
|
||||
("ingest_posts", tally.posts), ("ingest_failed", tally.failed),
|
||||
("ingest_slowest_ms", tally.slowest_ms), ("db_roundtrip_ms", roundtrip_ms),
|
||||
("unwatched_secs", unwatched_secs(last, now)),
|
||||
("restart_gap_secs", None if _started or last is None else (now - last).total_seconds()),
|
||||
("open_fds", fds), ("open_fds_pct", pct)]:
|
||||
if value is not None:
|
||||
batch.add(source_module="steward", resource_name="process",
|
||||
metric_name=metric_name, value=value, recorded_at=now)
|
||||
await batch.record(session)
|
||||
_started = True
|
||||
|
||||
if pct is not None and pct >= FD_WARN_PCT:
|
||||
if not _warned:
|
||||
|
||||
Reference in New Issue
Block a user