Files
FabledSteward/plugins/docker/routes.py
T
bvandeusen 82c3d2cf36
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 44s
CI / integration (push) Successful in 2m20s
CI / publish (push) Successful in 1m10s
feat(docker): per-container enrichment — health, restarts, exit code, I/O, grouping
First slice of milestone 77 (Docker monitoring depth). Surfaces real per-container
stats beyond basic state, all read-only on the existing push model.

- agent (→1.4.0): collect_docker now inspects each container (health, restart
  count, exit code, OOM) and reads net + block I/O from the stats payload; pulls
  compose project + swarm service/task/node from container labels. Per-container
  inspect+stats calls run over a small bounded ThreadPool so the ~1s-per-stats
  blocking doesn't stretch the sample on a busy host.
- schema (docker_003): additive columns on docker_containers — health, exit_code,
  oom_killed, compose_project, service_name, task_id, node_id, and BigInteger
  net/blk byte counters.
- ingest: persists the enrichment + restart_count (.get keeps older agents working).
- ui: Docker page rows now show health badge, uptime ("up 3d 4h"), restart count,
  exit code (+OOM) for stopped containers, and compose/service grouping label.
- tests: agent helpers (grouping, inspect fields, net/IO sum) + collect_docker
  assembly incl. inspect; integration asserts enrichment round-trips.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
2026-06-18 20:37:40 -04:00

261 lines
9.0 KiB
Python

# plugins/docker/routes.py
from __future__ import annotations
import json
from datetime import datetime, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import select
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.models.hosts import Host
from steward.core.time_range import parse_range, DEFAULT_RANGE
from .models import DockerContainer, DockerMetric
docker_bp = Blueprint("docker", __name__, template_folder="templates")
def _human_uptime(started_at: datetime | None) -> str | None:
"""Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m')."""
if started_at is None:
return None
if started_at.tzinfo is None:
started_at = started_at.replace(tzinfo=timezone.utc)
secs = int((datetime.now(timezone.utc) - started_at).total_seconds())
if secs < 0:
return None
d, rem = divmod(secs, 86400)
h, rem = divmod(rem, 3600)
m, _ = divmod(rem, 60)
if d:
return f"{d}d {h}h"
if h:
return f"{h}h {m}m"
return f"{m}m"
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
if len(values) < 2:
return f'<svg width="{width}" height="{height}"></svg>'
mn, mx = min(values), max(values)
if mx == mn:
mx = mn + 1.0
step = width / (len(values) - 1)
pts = []
for i, v in enumerate(values):
x = i * step
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
pts.append(f"{x:.1f},{y:.1f}")
poly = " ".join(pts)
return (
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
f'style="vertical-align:middle;">'
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
f'</svg>'
)
def _bucket(values: list[float], target: int = 40) -> list[float]:
"""Bucket-average a series down to ~target points (DB-agnostic, in Python).
Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
rows — averaging into a readable point count keeps the sparkline's shape.
"""
n = len(values)
if n <= target:
return values
size = (n + target - 1) // target
out: list[float] = []
for i in range(0, n, size):
chunk = values[i:i + size]
out.append(sum(chunk) / len(chunk))
return out
async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
"""Return (cpu_series, mem_series) sparkline-ready for one host's container."""
rows = (await db.execute(
select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
.where(DockerMetric.host_id == host_id)
.where(DockerMetric.container_name == name)
.where(DockerMetric.scraped_at >= since)
.order_by(DockerMetric.scraped_at)
)).all()
cpu = _bucket([r.cpu_pct or 0.0 for r in rows])
mem = _bucket([r.mem_pct or 0.0 for r in rows])
return cpu, mem
@docker_bp.get("/")
@require_role(UserRole.viewer)
async def index():
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
current_range = request.args.get("range", DEFAULT_RANGE)
return await render_template(
"docker/index.html",
poll_interval=poll_interval,
current_range=current_range,
)
async def _host_map(db, host_ids: set[str]) -> dict[str, Host]:
if not host_ids:
return {}
return {
h.id: h for h in (await db.execute(
select(Host).where(Host.id.in_(host_ids)))).scalars().all()
}
@docker_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
"""HTMX fragment: containers grouped by host, with resource sparklines."""
since, range_key = parse_range(request.args.get("range"))
async with current_app.db_sessionmaker() as db:
containers = list((await db.execute(
select(DockerContainer).order_by(
(DockerContainer.status == "running").desc(),
DockerContainer.name,
)
)).scalars())
hosts = await _host_map(db, {c.host_id for c in containers})
# Group by host so each container is clearly attributed to the box it
# runs on (names are only unique within a host now).
groups: dict[str, dict] = {}
for c in containers:
g = groups.get(c.host_id)
if g is None:
host = hosts.get(c.host_id)
g = groups[c.host_id] = {
"host": host,
"host_id": c.host_id,
"host_name": host.name if host else c.host_id,
"containers": [], "running": 0, "stopped": 0,
}
cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since)
g["containers"].append({
"container": c,
"ports": json.loads(c.ports_json) if c.ports_json else [],
"uptime": _human_uptime(c.started_at) if c.status == "running" else None,
"sparkline_cpu": _sparkline(cpu_hist),
"sparkline_mem": _sparkline(mem_hist),
})
if c.status == "running":
g["running"] += 1
else:
g["stopped"] += 1
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
running = sum(g["running"] for g in host_groups)
total = len(containers)
return await render_template(
"docker/rows.html",
host_groups=host_groups,
running=running,
stopped=total - running,
total=total,
range_key=range_key,
)
@docker_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""HTMX dashboard widget: container status overview, grouped by host."""
show_stopped = request.args.get("show_stopped", "no") == "yes"
widget_id = request.args.get("wid", "0")
async with current_app.db_sessionmaker() as db:
all_containers = list((await db.execute(
select(DockerContainer).order_by(
(DockerContainer.status == "running").desc(),
DockerContainer.name,
)
)).scalars())
hosts = await _host_map(db, {c.host_id for c in all_containers})
running = [c for c in all_containers if c.status == "running"]
stopped = [c for c in all_containers if c.status != "running"]
display = all_containers if show_stopped else running
# Group for display so multi-host fleets read clearly; single-host stays flat.
groups: dict[str, dict] = {}
for c in display:
g = groups.setdefault(c.host_id, {
"host_id": c.host_id,
"host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
"containers": [],
})
g["containers"].append(c)
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
return await render_template(
"docker/widget.html",
host_groups=host_groups,
multi_host=len(host_groups) > 1,
running_count=len(running),
stopped_count=len(stopped),
show_stopped=show_stopped,
widget_id=widget_id,
)
@docker_bp.get("/widget/resources")
@require_role(UserRole.viewer)
async def widget_resources():
"""HTMX dashboard widget: CPU + memory usage for the busiest containers."""
limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
widget_id = request.args.get("wid", "0")
async with current_app.db_sessionmaker() as db:
containers = list((await db.execute(
select(DockerContainer)
.where(DockerContainer.status == "running")
.order_by(DockerContainer.cpu_pct.desc().nullslast())
)).scalars())[:limit]
hosts = await _host_map(db, {c.host_id for c in containers})
rows_data = [
{"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
for c in containers
]
return await render_template(
"docker/widget_resources.html",
rows=rows_data,
multi_host=len({c.host_id for c in containers}) > 1,
widget_id=widget_id,
)
@docker_bp.get("/host/<host_id>")
@require_role(UserRole.viewer)
async def host_panel(host_id: str):
"""Per-host Docker fragment embedded on the core Hosts hub via HTMX.
Renders nothing when the host reports no containers, so hosts without Docker
don't carry an empty card.
"""
async with current_app.db_sessionmaker() as db:
containers = list((await db.execute(
select(DockerContainer)
.where(DockerContainer.host_id == host_id)
.order_by(
(DockerContainer.status == "running").desc(),
DockerContainer.name,
)
)).scalars())
if not containers:
return ""
running = sum(1 for c in containers if c.status == "running")
return await render_template(
"docker/host_panel.html",
containers=containers,
running=running,
stopped=len(containers) - running,
)