From acc487f9c3f67333226644cbc6c43f0cf0a243cc Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 30 Oct 2026 20:01:10 -0400 Subject: [PATCH] fix(docker): the move of task-named history rests between passes (#6718) #6718 "A swarm container's metrics, normal and anomalies are keyed by its task name, so they start over whenever the task moves". Live, the move took VDNT-Docker02's I/O pressure (psi_io_full_avg60) from about 1% to about 23%, and passes of 5000 raw samples outran their 15 s slot. Each moved sample rewrites its row and every index on it. Batches are 1000 samples, and after a pass the task waits four times as long as it took, so it holds the disk for about a fifth of the time and the move slows if the disk does. Co-Authored-By: Claude Opus 5.5 --- plugins/docker/rekey.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/docker/rekey.py b/plugins/docker/rekey.py index 95687fd..3fe0b1d 100644 --- a/plugins/docker/rekey.py +++ b/plugins/docker/rekey.py @@ -32,9 +32,10 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -BATCH = 5000 # raw samples moved in one go +BATCH = 1000 # raw samples moved in one go: each rewrites its row and every index on it EVERY_S = 15 PASS_S = 2 # a key with only a few rows takes milliseconds, and there can be many +REST = 4 # after a pass, wait this many times as long as it took, so ingest keeps the disk TRAFFIC = ("net_rx_bps", "net_tx_bps") _TASK_KEY = r"^[^/]+/.+\.([0-9]+|[a-z0-9]{25})\.[a-z0-9]{25}$" _NOT_TRAFFIC = "metric_name NOT IN ('net_rx_bps', 'net_tx_bps')" @@ -167,10 +168,12 @@ async def move_key(session, old: str) -> bool: def make_rekey_task(app: "Quart") -> ScheduledTask: remaining: list[str] | None = None busy = asyncio.Lock() + resume_at = 0.0 async def rekey() -> None: - nonlocal remaining - if remaining == [] or busy.locked(): + nonlocal remaining, resume_at + started = time.monotonic() + if remaining == [] or busy.locked() or started < resume_at: return async with busy, app.db_sessionmaker() as session: if remaining is None: @@ -185,5 +188,7 @@ def make_rekey_task(app: "Quart") -> ScheduledTask: remaining.pop() if not remaining: logger.info("Swarm history under task names: all moved") + now = time.monotonic() + resume_at = now + REST * (now - started) return ScheduledTask(name="docker_rekey", coro_factory=rekey, interval_seconds=EVERY_S)