diff --git a/plugins/docker/rekey.py b/plugins/docker/rekey.py index 95687fd..3fe0b1d 100644 --- a/plugins/docker/rekey.py +++ b/plugins/docker/rekey.py @@ -32,9 +32,10 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -BATCH = 5000 # raw samples moved in one go +BATCH = 1000 # raw samples moved in one go: each rewrites its row and every index on it EVERY_S = 15 PASS_S = 2 # a key with only a few rows takes milliseconds, and there can be many +REST = 4 # after a pass, wait this many times as long as it took, so ingest keeps the disk TRAFFIC = ("net_rx_bps", "net_tx_bps") _TASK_KEY = r"^[^/]+/.+\.([0-9]+|[a-z0-9]{25})\.[a-z0-9]{25}$" _NOT_TRAFFIC = "metric_name NOT IN ('net_rx_bps', 'net_tx_bps')" @@ -167,10 +168,12 @@ async def move_key(session, old: str) -> bool: def make_rekey_task(app: "Quart") -> ScheduledTask: remaining: list[str] | None = None busy = asyncio.Lock() + resume_at = 0.0 async def rekey() -> None: - nonlocal remaining - if remaining == [] or busy.locked(): + nonlocal remaining, resume_at + started = time.monotonic() + if remaining == [] or busy.locked() or started < resume_at: return async with busy, app.db_sessionmaker() as session: if remaining is None: @@ -185,5 +188,7 @@ def make_rekey_task(app: "Quart") -> ScheduledTask: remaining.pop() if not remaining: logger.info("Swarm history under task names: all moved") + now = time.monotonic() + resume_at = now + REST * (now - started) return ScheduledTask(name="docker_rekey", coro_factory=rekey, interval_seconds=EVERY_S)