fix(docker): the slow-ingest line names which part of the Docker persist took the time (#5181)
persist_host_docker marks docker.status, .swarm, .disk, .logs, .metrics, .events, .containers, .alerts and .reap on the ingest's phase timer, each flushed before its mark so a part's writes are charged to it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -289,7 +289,7 @@ async def _expire_swarm(session, host) -> None:
|
||||
|
||||
|
||||
async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
logs=None, status=None) -> None:
|
||||
logs=None, status=None, phases=None) -> None:
|
||||
"""Upsert containers + time-series + lifecycle events + swarm for one host.
|
||||
|
||||
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
|
||||
@@ -305,20 +305,33 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
|
||||
"ok" with an empty snapshot clears the host's containers, and a node that
|
||||
says it is no longer a manager loses the swarm rows it reported as one.
|
||||
|
||||
`phases`, when given, is marked after each part (docker.status, .swarm,
|
||||
.disk, .logs, .metrics, .events, .containers, .alerts, .reap), each
|
||||
flushed before its mark, so a slow ingest names the part (#5181).
|
||||
"""
|
||||
from steward.core.alerts import MetricBatch
|
||||
from .models import DockerContainer, DockerEvent, DockerMetric
|
||||
|
||||
async def mark(name: str) -> None:
|
||||
if phases is not None:
|
||||
await session.flush()
|
||||
phases.mark(f"docker.{name}")
|
||||
|
||||
if status is not None:
|
||||
await _persist_status(session, host, status)
|
||||
if status.get("swarm_manager") is False:
|
||||
await _expire_swarm(session, host)
|
||||
await mark("status")
|
||||
if swarm is not None:
|
||||
await _persist_swarm(session, host, swarm)
|
||||
await mark("swarm")
|
||||
if disk is not None:
|
||||
await _persist_disk(session, host, disk)
|
||||
await mark("disk")
|
||||
if logs:
|
||||
await _persist_logs(session, host, logs)
|
||||
await mark("logs")
|
||||
|
||||
if not snapshots:
|
||||
return
|
||||
@@ -337,6 +350,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
mem_pct=c.get("mem_pct") or 0.0,
|
||||
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
|
||||
))
|
||||
await mark("metrics")
|
||||
|
||||
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
|
||||
# used to diff lifecycle events. Skip event derivation on the host's first
|
||||
@@ -356,6 +370,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
host_id=host.id, container_name=name,
|
||||
event=event, detail=detail, at=latest_at,
|
||||
))
|
||||
await mark("events")
|
||||
|
||||
# Current state + alerts from the newest snapshot only.
|
||||
batch = MetricBatch()
|
||||
@@ -423,7 +438,9 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
value=1.0 if health == "healthy" else 0.0,
|
||||
)
|
||||
|
||||
await mark("containers")
|
||||
await batch.record(session)
|
||||
await mark("alerts")
|
||||
|
||||
# Reap containers that vanished from the host's latest listing. Without this,
|
||||
# removed one-shot containers (CI job runners, buildkit builders, codex jobs)
|
||||
@@ -437,3 +454,4 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
if seen_names:
|
||||
reap = reap.where(DockerContainer.name.notin_(seen_names))
|
||||
await session.execute(reap)
|
||||
await mark("reap")
|
||||
|
||||
Reference in New Issue
Block a user