# plugins/docker/ingest.py """Persist host-scoped Docker samples pushed by the host agent. Published as the "docker.persist_host_samples" capability (see __init__.setup), so the host_agent plugin can hand off a sample's `docker` array (and, on a swarm manager, its `swarm` object) WITHOUT importing the docker models — the coupling is opportunistic and degrades to a no-op when the docker plugin is disabled. Runs inside the caller's open transaction (the ingest handler's SAVEPOINT); never opens or commits its own. """ from __future__ import annotations import json from datetime import datetime from sqlalchemy import delete, select # Stopped/terminal container states (anything not "running"/"paused"/"restarting"). _STOPPED_STATES = {"exited", "dead", "stopped"} def _parse_started_at(value) -> datetime | None: """Parse the agent's ISO-8601 started_at string back to a datetime.""" if not isinstance(value, str) or not value: return None try: return datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: return None def _derive_events(old_state: dict, new_containers: list) -> list: """Diff a fresh snapshot against stored per-container state → lifecycle events. Pure function (no DB) so it's unit-testable. `old_state` maps name → the previously stored {status, health, oom_killed, exit_code}; `new_containers` is the newest snapshot's list of container dicts. Returns a list of (container_name, event, detail) tuples: start — a container transitions into running (or a genuinely new container appears already running) stop — running → not-running, or a running container is removed die — same as stop but with a non-zero exit code (abnormal) oom — OOMKilled flips False→True health_change — HEALTHCHECK status string changes The CALLER skips this entirely on a host's first-ever snapshot (empty old_state) so the baseline doesn't emit a spurious "start" per existing container — a later-appearing container still gets one because by then old_state is populated and that container's prior entry is simply absent. """ events: list = [] new_by_name = {c["name"]: c for c in new_containers if c.get("name")} for name, c in new_by_name.items(): new_status = (c.get("status") or "").lower() new_running = new_status == "running" new_oom = bool(c.get("oom_killed")) new_health = c.get("health") new_exit = c.get("exit_code") old = old_state.get(name) if old is None: # Newly observed container — only a start is meaningful (we have no # prior state to diff a death against). if new_running: events.append((name, "start", c.get("image") or None)) continue old_running = (old.get("status") or "").lower() == "running" if new_running and not old_running: events.append((name, "start", c.get("image") or None)) elif old_running and not new_running: if new_exit not in (None, 0): events.append((name, "die", f"exit {new_exit}")) else: events.append((name, "stop", None)) if new_oom and not bool(old.get("oom_killed")): events.append((name, "oom", None)) if new_health != old.get("health") and (new_health or old.get("health")): events.append((name, "health_change", f"{old.get('health') or '?'}→{new_health or '?'}")) # A running container that vanished from the listing entirely (removed). for name, old in old_state.items(): if name not in new_by_name and (old.get("status") or "").lower() == "running": events.append((name, "stop", "removed")) return events async def _persist_swarm(session, host, swarm: dict) -> None: """Upsert this manager's swarm topology; drop rows no longer reported. Current-state tables (not time-series): a manager re-reports its full services/nodes set every sample, so we upsert what's present and delete what isn't (scoped to this host so two managers don't clobber each other). """ from datetime import timezone from .models import DockerSwarmNode, DockerSwarmService now = datetime.now(timezone.utc) services = swarm.get("services") or [] seen_services: set[str] = set() for s in services: name = s.get("service_name") if not name: continue seen_services.add(name) row = await session.get(DockerSwarmService, (host.id, name)) if row is None: row = DockerSwarmService(host_id=host.id, service_name=name) session.add(row) row.mode = s.get("mode") or "replicated" row.desired = int(s.get("desired") or 0) row.running = int(s.get("running") or 0) row.image = s.get("image") row.placement_json = json.dumps(s.get("placement") or []) row.updated_at = now stale_services = delete(DockerSwarmService).where( DockerSwarmService.host_id == host.id) if seen_services: stale_services = stale_services.where( DockerSwarmService.service_name.notin_(seen_services)) await session.execute(stale_services) nodes = swarm.get("nodes") or [] seen_nodes: set[str] = set() for n in nodes: nid = n.get("node_id") if not nid: continue seen_nodes.add(nid) row = await session.get(DockerSwarmNode, (host.id, nid)) if row is None: row = DockerSwarmNode(host_id=host.id, node_id=nid) session.add(row) row.hostname = n.get("hostname") or "" row.role = n.get("role") or "worker" row.availability = n.get("availability") or "active" row.status = n.get("status") or "unknown" row.leader = bool(n.get("leader", False)) row.updated_at = now stale_nodes = delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id) if seen_nodes: stale_nodes = stale_nodes.where(DockerSwarmNode.node_id.notin_(seen_nodes)) await session.execute(stale_nodes) async def persist_host_docker(session, host, snapshots, swarm=None) -> None: """Upsert containers + time-series + lifecycle events + swarm for one host. `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) — one entry per ingested sample carrying docker data (usually one; more when the agent flushes a backlog). Every snapshot contributes time-series DockerMetric rows; the newest snapshot drives current container state, the alert pipeline, and lifecycle-event derivation. `swarm` is the newest sample's swarm object (or None off managers) — persisted when present. """ from steward.core.alerts import record_metric from .models import DockerContainer, DockerEvent, DockerMetric if swarm is not None: await _persist_swarm(session, host, swarm) if not snapshots: return ordered = sorted(snapshots, key=lambda s: s[0]) latest_at, latest_containers = ordered[-1] # Time-series points for every snapshot (running containers with a CPU read). for recorded_at, containers in ordered: for c in containers: if c.get("status") == "running" and c.get("cpu_pct") is not None: session.add(DockerMetric( host_id=host.id, container_name=c["name"], scraped_at=recorded_at, cpu_pct=c["cpu_pct"], mem_pct=c.get("mem_pct") or 0.0, mem_usage_bytes=c.get("mem_usage_bytes") or 0, )) # Snapshot of stored per-container state BEFORE the upsert overwrites it — # used to diff lifecycle events. Skip event derivation on the host's first # snapshot (empty state) so we don't emit a start per pre-existing container. old_rows = (await session.execute( select(DockerContainer).where(DockerContainer.host_id == host.id) )).scalars().all() old_state = { r.name: {"status": r.status, "health": r.health, "oom_killed": r.oom_killed, "exit_code": r.exit_code} for r in old_rows } if old_state: for name, event, detail in _derive_events(old_state, latest_containers): session.add(DockerEvent( host_id=host.id, container_name=name, event=event, detail=detail, at=latest_at, )) # Current state + alerts from the newest snapshot only. for c in latest_containers: name = c.get("name") if not name: continue existing = await session.get(DockerContainer, (host.id, name)) if existing is None: existing = DockerContainer(host_id=host.id, name=name) session.add(existing) existing.container_id = c.get("container_id", "") or "" existing.image = c.get("image", "") or "" existing.status = c.get("status", "unknown") or "unknown" existing.cpu_pct = c.get("cpu_pct") existing.mem_usage_bytes = c.get("mem_usage_bytes") existing.mem_limit_bytes = c.get("mem_limit_bytes") existing.mem_pct = c.get("mem_pct") existing.ports_json = json.dumps(c.get("ports") or []) existing.started_at = _parse_started_at(c.get("started_at")) existing.scraped_at = latest_at # Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields # stay None/0 when absent from the payload). existing.restart_count = c.get("restart_count", 0) or 0 existing.health = c.get("health") existing.exit_code = c.get("exit_code") existing.oom_killed = bool(c.get("oom_killed", False)) existing.compose_project = c.get("compose_project") existing.service_name = c.get("service_name") existing.task_id = c.get("task_id") existing.node_id = c.get("node_id") existing.net_rx_bytes = c.get("net_rx_bytes") existing.net_tx_bytes = c.get("net_tx_bytes") existing.blk_read_bytes = c.get("blk_read_bytes") existing.blk_write_bytes = c.get("blk_write_bytes") # Alert pipeline — resource is host-scoped so containers of the same name # on different hosts don't collide in the metric/alert namespace. resource = f"{host.name}/{name}" if c.get("status") == "running" and c.get("cpu_pct") is not None: await record_metric( session=session, source_module="docker", resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"], ) if c.get("mem_pct") is not None: await record_metric( session=session, source_module="docker", resource_name=resource, metric_name="mem_pct", value=c["mem_pct"], ) # Restart count is alertable regardless of state (crash-looping matters # most while the container is down/restarting). await record_metric( session=session, source_module="docker", resource_name=resource, metric_name="restart_count", value=float(c.get("restart_count", 0) or 0), ) # Health → 1.0/0.0 only for containers that actually define a HEALTHCHECK # (health is None otherwise — recording 0 would false-alarm every plain # container). health = c.get("health") if health in ("healthy", "unhealthy", "starting"): await record_metric( session=session, source_module="docker", resource_name=resource, metric_name="is_healthy", value=1.0 if health == "healthy" else 0.0, )