fix(docker): the move of task-named history finishes many small keys in one pass (#6718)
#6718 "A swarm container's metrics, normal and anomalies are keyed by its task name, so they start over whenever the task moves". Live, the move found 391,440 keys, about 2,500 of them with raw samples, and took one key per 15 s pass: about 68 days, in sorted order, so recent history could wait months while charts and normals read thin. A pass now runs for up to 2 s, and keys with raw samples go first. Old hourly keys take a few statements each. A key with no anomalies no longer rewrites the service's timeline entries. The start logs how many keys each table holds, as that count is far higher than expected and not explained yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -121,6 +121,40 @@ def test_a_tasks_history_moves_onto_its_service_a_batch_at_a_time(app, monkeypat
|
||||
assert [sp.anomaly_id for sp in spans] == [row.id]
|
||||
|
||||
|
||||
def test_keys_with_raw_samples_are_taken_first_and_a_pass_finishes_many_small_keys(app):
|
||||
"""Taken from the end of old_keys: a chart and a normal read recent history,
|
||||
so a key with raw samples doesn't wait behind thousands that only have old
|
||||
hours. One pass moves every small key it has time for."""
|
||||
from sqlalchemy import select
|
||||
from steward.models.metrics import PluginMetricHourly, PluginMetricLatest
|
||||
rekey = _docker("rekey")
|
||||
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
aged = [f"nas/it{tag}_old.1.{c * 25}" for c in "bcd"]
|
||||
recent = f"nas/it{tag}_new.1.{TASK}"
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add_all([_hour(k, "cpu_pct", NOW, avg=1.0, top=2.0) for k in aged])
|
||||
s.add(PluginMetricLatest(source_module="docker", resource_name=recent, metric_name="cpu_pct",
|
||||
value=1.0, recorded_at=NOW))
|
||||
async with s.begin():
|
||||
found = await rekey.old_keys(s)
|
||||
await rekey.make_rekey_task(app).coro_factory()
|
||||
async with app.db_sessionmaker() as s:
|
||||
left = (await s.execute(select(PluginMetricHourly.resource_name).where(
|
||||
PluginMetricHourly.resource_name.in_(aged)))).scalars().all()
|
||||
moved = (await s.execute(select(PluginMetricHourly.value_avg).where(
|
||||
PluginMetricHourly.resource_name == f"swarm/it{tag}_old"))).scalars().all()
|
||||
return found, left, moved
|
||||
|
||||
found, left, moved = asyncio.run(_go())
|
||||
assert all(found.index(recent) > found.index(k) for k in aged)
|
||||
assert left == []
|
||||
assert moved == [1.0]
|
||||
|
||||
|
||||
def test_an_alert_rule_on_a_task_follows_it_to_its_service(app):
|
||||
from sqlalchemy import select
|
||||
from steward.models.alerts import AlertOperator, AlertRule
|
||||
|
||||
Reference in New Issue
Block a user