bvandeusen
82c3d2cf36
First slice of milestone 77 (Docker monitoring depth). Surfaces real per-container
stats beyond basic state, all read-only on the existing push model.
- agent (→1.4.0): collect_docker now inspects each container (health, restart
count, exit code, OOM) and reads net + block I/O from the stats payload; pulls
compose project + swarm service/task/node from container labels. Per-container
inspect+stats calls run over a small bounded ThreadPool so the ~1s-per-stats
blocking doesn't stretch the sample on a busy host.
- schema (docker_003): additive columns on docker_containers — health, exit_code,
oom_killed, compose_project, service_name, task_id, node_id, and BigInteger
net/blk byte counters.
- ingest: persists the enrichment + restart_count (.get keeps older agents working).
- ui: Docker page rows now show health badge, uptime ("up 3d 4h"), restart count,
exit code (+OOM) for stopped containers, and compose/service grouping label.
- tests: agent helpers (grouping, inspect fields, net/IO sum) + collect_docker
assembly incl. inspect; integration asserts enrichment round-trips.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
104 lines
4.6 KiB
Python
104 lines
4.6 KiB
Python
# plugins/docker/ingest.py
|
|
"""Persist host-scoped Docker samples pushed by the host agent.
|
|
|
|
Published as the "docker.persist_host_samples" capability (see __init__.setup),
|
|
so the host_agent plugin can hand off the `docker` array from a sample WITHOUT
|
|
importing the docker models — the coupling is opportunistic and degrades to a
|
|
no-op when the docker plugin is disabled. Runs inside the caller's open
|
|
transaction (the ingest handler's session); never opens or commits its own.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
|
|
|
|
def _parse_started_at(value) -> datetime | None:
|
|
"""Parse the agent's ISO-8601 started_at string back to a datetime."""
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
async def persist_host_docker(session, host, snapshots) -> None:
|
|
"""Upsert containers + append time-series for one host's docker snapshots.
|
|
|
|
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
|
|
one entry per ingested sample carrying docker data (usually just one; more
|
|
when the agent flushes a backlog). Every snapshot contributes time-series
|
|
DockerMetric rows; the newest snapshot drives current container state and
|
|
the alert pipeline (record_metric writes "now", so feeding it stale buffered
|
|
snapshots would be misleading).
|
|
"""
|
|
from steward.core.alerts import record_metric
|
|
from .models import DockerContainer, DockerMetric
|
|
|
|
if not snapshots:
|
|
return
|
|
ordered = sorted(snapshots, key=lambda s: s[0])
|
|
latest_at, latest_containers = ordered[-1]
|
|
|
|
# Time-series points for every snapshot (running containers with a CPU read).
|
|
for recorded_at, containers in ordered:
|
|
for c in containers:
|
|
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
|
session.add(DockerMetric(
|
|
host_id=host.id,
|
|
container_name=c["name"],
|
|
scraped_at=recorded_at,
|
|
cpu_pct=c["cpu_pct"],
|
|
mem_pct=c.get("mem_pct") or 0.0,
|
|
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
|
|
))
|
|
|
|
# Current state + alerts from the newest snapshot only.
|
|
for c in latest_containers:
|
|
name = c.get("name")
|
|
if not name:
|
|
continue
|
|
existing = await session.get(DockerContainer, (host.id, name))
|
|
if existing is None:
|
|
existing = DockerContainer(host_id=host.id, name=name)
|
|
session.add(existing)
|
|
existing.container_id = c.get("container_id", "") or ""
|
|
existing.image = c.get("image", "") or ""
|
|
existing.status = c.get("status", "unknown") or "unknown"
|
|
existing.cpu_pct = c.get("cpu_pct")
|
|
existing.mem_usage_bytes = c.get("mem_usage_bytes")
|
|
existing.mem_limit_bytes = c.get("mem_limit_bytes")
|
|
existing.mem_pct = c.get("mem_pct")
|
|
existing.ports_json = json.dumps(c.get("ports") or [])
|
|
existing.started_at = _parse_started_at(c.get("started_at"))
|
|
existing.scraped_at = latest_at
|
|
# Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields
|
|
# stay None/0 when absent from the payload).
|
|
existing.restart_count = c.get("restart_count", 0) or 0
|
|
existing.health = c.get("health")
|
|
existing.exit_code = c.get("exit_code")
|
|
existing.oom_killed = bool(c.get("oom_killed", False))
|
|
existing.compose_project = c.get("compose_project")
|
|
existing.service_name = c.get("service_name")
|
|
existing.task_id = c.get("task_id")
|
|
existing.node_id = c.get("node_id")
|
|
existing.net_rx_bytes = c.get("net_rx_bytes")
|
|
existing.net_tx_bytes = c.get("net_tx_bytes")
|
|
existing.blk_read_bytes = c.get("blk_read_bytes")
|
|
existing.blk_write_bytes = c.get("blk_write_bytes")
|
|
|
|
# Alert pipeline — resource is host-scoped so containers of the same name
|
|
# on different hosts don't collide in the metric/alert namespace.
|
|
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
|
resource = f"{host.name}/{name}"
|
|
await record_metric(
|
|
session=session, source_module="docker",
|
|
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
|
|
)
|
|
if c.get("mem_pct") is not None:
|
|
await record_metric(
|
|
session=session, source_module="docker",
|
|
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
|
|
)
|