fix(metrics): seed the latest table in the background, not in the migration (#5208)
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 1m7s
CI / unit (push) Successful in 1m23s
CI / publish (push) Successful in 1m0s

Migration 0030 seeded plugin_metric_latest from the whole history: two index
probes per distinct metric key. On the live instance that held the first
boot for 7+ minutes with Steward down. The migration now only creates the
table. Live sources fill it on their next sample, and backfill_latest runs
after startup for resources that have history but no latest rows. Per
resource it does one backward probe for the last sample, then takes the
newest value per metric within the hour before it. Resources that already
have rows are skipped, so later boots only walk the resource names.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 23:19:38 -04:00
co-authored by Claude Opus 5.5
parent 1845ba973c
commit 8941ed0372
4 changed files with 100 additions and 26 deletions
+36
View File
@@ -148,3 +148,39 @@ def test_a_renamed_host_carries_its_latest_values(app):
assert old_latest == {}
assert new_latest["cpu_pct"][0] == 5.0
assert new_root["disk_used_pct"][0] == 60.0
def test_the_backfill_fills_a_quiet_resource_once_and_leaves_live_ones_alone(app):
import logging
from steward.core.alerts import record_metrics
from steward.core.metric_series import backfill_latest
now = datetime.now(timezone.utc)
quiet, live = ("bf-quiet-" + uuid.uuid4().hex[:8]), ("bf-live-" + uuid.uuid4().hex[:8])
went_quiet = now - timedelta(hours=3)
async def go():
async with app.db_sessionmaker() as s:
async with s.begin():
# History from before the table existed: rows with no latest.
s.add_all([
_sample(quiet, "cpu_pct", 10.0, went_quiet - timedelta(minutes=30)),
_sample(quiet, "cpu_pct", 11.0, went_quiet),
_sample(quiet, "mem_used_pct", 50.0, went_quiet - timedelta(minutes=10)),
_sample(quiet, "load_1m", 9.9, went_quiet - timedelta(hours=2)), # outside the hour
])
async with s.begin():
await record_metrics(s, [_sample(live, "cpu_pct", 70.0, now)])
s.add(_sample(live, "cpu_pct", 1.0, now - timedelta(minutes=5)))
first = await backfill_latest(app.db_sessionmaker, logging.getLogger("test"))
second = await backfill_latest(app.db_sessionmaker, logging.getLogger("test"))
async with app.db_sessionmaker() as s:
return first, second, await _latest(s, quiet), await _latest(s, live)
first, second, quiet_latest, live_latest = asyncio.run(go())
assert first >= 2 and second == 0
assert quiet_latest["cpu_pct"][0] == 11.0
assert quiet_latest["mem_used_pct"][0] == 50.0
assert "load_1m" not in quiet_latest
assert live_latest == {"cpu_pct": (70.0, live_latest["cpu_pct"][1])}