fix(metrics): seed the latest table in the background, not in the migration (#5208)
Migration 0030 seeded plugin_metric_latest from the whole history: two index probes per distinct metric key. On the live instance that held the first boot for 7+ minutes with Steward down. The migration now only creates the table. Live sources fill it on their next sample, and backfill_latest runs after startup for resources that have history but no latest rows. Per resource it does one backward probe for the last sample, then takes the newest value per metric within the hour before it. Resources that already have rows are skipped, so later boots only walk the resource names. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -148,3 +148,39 @@ def test_a_renamed_host_carries_its_latest_values(app):
|
||||
assert old_latest == {}
|
||||
assert new_latest["cpu_pct"][0] == 5.0
|
||||
assert new_root["disk_used_pct"][0] == 60.0
|
||||
|
||||
|
||||
def test_the_backfill_fills_a_quiet_resource_once_and_leaves_live_ones_alone(app):
|
||||
import logging
|
||||
|
||||
from steward.core.alerts import record_metrics
|
||||
from steward.core.metric_series import backfill_latest
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
quiet, live = ("bf-quiet-" + uuid.uuid4().hex[:8]), ("bf-live-" + uuid.uuid4().hex[:8])
|
||||
went_quiet = now - timedelta(hours=3)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
# History from before the table existed: rows with no latest.
|
||||
s.add_all([
|
||||
_sample(quiet, "cpu_pct", 10.0, went_quiet - timedelta(minutes=30)),
|
||||
_sample(quiet, "cpu_pct", 11.0, went_quiet),
|
||||
_sample(quiet, "mem_used_pct", 50.0, went_quiet - timedelta(minutes=10)),
|
||||
_sample(quiet, "load_1m", 9.9, went_quiet - timedelta(hours=2)), # outside the hour
|
||||
])
|
||||
async with s.begin():
|
||||
await record_metrics(s, [_sample(live, "cpu_pct", 70.0, now)])
|
||||
s.add(_sample(live, "cpu_pct", 1.0, now - timedelta(minutes=5)))
|
||||
first = await backfill_latest(app.db_sessionmaker, logging.getLogger("test"))
|
||||
second = await backfill_latest(app.db_sessionmaker, logging.getLogger("test"))
|
||||
async with app.db_sessionmaker() as s:
|
||||
return first, second, await _latest(s, quiet), await _latest(s, live)
|
||||
|
||||
first, second, quiet_latest, live_latest = asyncio.run(go())
|
||||
assert first >= 2 and second == 0
|
||||
assert quiet_latest["cpu_pct"][0] == 11.0
|
||||
assert quiet_latest["mem_used_pct"][0] == 50.0
|
||||
assert "load_1m" not in quiet_latest
|
||||
assert live_latest == {"cpu_pct": (70.0, live_latest["cpu_pct"][1])}
|
||||
|
||||
Reference in New Issue
Block a user