82c3d2cf36
First slice of milestone 77 (Docker monitoring depth). Surfaces real per-container
stats beyond basic state, all read-only on the existing push model.
- agent (→1.4.0): collect_docker now inspects each container (health, restart
count, exit code, OOM) and reads net + block I/O from the stats payload; pulls
compose project + swarm service/task/node from container labels. Per-container
inspect+stats calls run over a small bounded ThreadPool so the ~1s-per-stats
blocking doesn't stretch the sample on a busy host.
- schema (docker_003): additive columns on docker_containers — health, exit_code,
oom_killed, compose_project, service_name, task_id, node_id, and BigInteger
net/blk byte counters.
- ingest: persists the enrichment + restart_count (.get keeps older agents working).
- ui: Docker page rows now show health badge, uptime ("up 3d 4h"), restart count,
exit code (+OOM) for stopped containers, and compose/service grouping label.
- tests: agent helpers (grouping, inspect fields, net/IO sum) + collect_docker
assembly incl. inspect; integration asserts enrichment round-trips.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
261 lines
9.0 KiB
Python
261 lines
9.0 KiB
Python
# plugins/docker/routes.py
|
|
from __future__ import annotations
|
|
import json
|
|
from datetime import datetime, timezone
|
|
|
|
from quart import Blueprint, current_app, render_template, request
|
|
from sqlalchemy import select
|
|
|
|
from steward.auth.middleware import require_role
|
|
from steward.models.users import UserRole
|
|
from steward.models.hosts import Host
|
|
from steward.core.time_range import parse_range, DEFAULT_RANGE
|
|
from .models import DockerContainer, DockerMetric
|
|
|
|
docker_bp = Blueprint("docker", __name__, template_folder="templates")
|
|
|
|
|
|
def _human_uptime(started_at: datetime | None) -> str | None:
|
|
"""Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m')."""
|
|
if started_at is None:
|
|
return None
|
|
if started_at.tzinfo is None:
|
|
started_at = started_at.replace(tzinfo=timezone.utc)
|
|
secs = int((datetime.now(timezone.utc) - started_at).total_seconds())
|
|
if secs < 0:
|
|
return None
|
|
d, rem = divmod(secs, 86400)
|
|
h, rem = divmod(rem, 3600)
|
|
m, _ = divmod(rem, 60)
|
|
if d:
|
|
return f"{d}d {h}h"
|
|
if h:
|
|
return f"{h}h {m}m"
|
|
return f"{m}m"
|
|
|
|
|
|
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
|
|
if len(values) < 2:
|
|
return f'<svg width="{width}" height="{height}"></svg>'
|
|
mn, mx = min(values), max(values)
|
|
if mx == mn:
|
|
mx = mn + 1.0
|
|
step = width / (len(values) - 1)
|
|
pts = []
|
|
for i, v in enumerate(values):
|
|
x = i * step
|
|
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
|
|
pts.append(f"{x:.1f},{y:.1f}")
|
|
poly = " ".join(pts)
|
|
return (
|
|
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
|
|
f'style="vertical-align:middle;">'
|
|
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
|
|
f'</svg>'
|
|
)
|
|
|
|
|
|
def _bucket(values: list[float], target: int = 40) -> list[float]:
|
|
"""Bucket-average a series down to ~target points (DB-agnostic, in Python).
|
|
|
|
Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
|
|
on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
|
|
rows — averaging into a readable point count keeps the sparkline's shape.
|
|
"""
|
|
n = len(values)
|
|
if n <= target:
|
|
return values
|
|
size = (n + target - 1) // target
|
|
out: list[float] = []
|
|
for i in range(0, n, size):
|
|
chunk = values[i:i + size]
|
|
out.append(sum(chunk) / len(chunk))
|
|
return out
|
|
|
|
|
|
async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
|
|
"""Return (cpu_series, mem_series) sparkline-ready for one host's container."""
|
|
rows = (await db.execute(
|
|
select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
|
|
.where(DockerMetric.host_id == host_id)
|
|
.where(DockerMetric.container_name == name)
|
|
.where(DockerMetric.scraped_at >= since)
|
|
.order_by(DockerMetric.scraped_at)
|
|
)).all()
|
|
cpu = _bucket([r.cpu_pct or 0.0 for r in rows])
|
|
mem = _bucket([r.mem_pct or 0.0 for r in rows])
|
|
return cpu, mem
|
|
|
|
|
|
@docker_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def index():
|
|
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
|
current_range = request.args.get("range", DEFAULT_RANGE)
|
|
return await render_template(
|
|
"docker/index.html",
|
|
poll_interval=poll_interval,
|
|
current_range=current_range,
|
|
)
|
|
|
|
|
|
async def _host_map(db, host_ids: set[str]) -> dict[str, Host]:
|
|
if not host_ids:
|
|
return {}
|
|
return {
|
|
h.id: h for h in (await db.execute(
|
|
select(Host).where(Host.id.in_(host_ids)))).scalars().all()
|
|
}
|
|
|
|
|
|
@docker_bp.get("/rows")
|
|
@require_role(UserRole.viewer)
|
|
async def rows():
|
|
"""HTMX fragment: containers grouped by host, with resource sparklines."""
|
|
since, range_key = parse_range(request.args.get("range"))
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
containers = list((await db.execute(
|
|
select(DockerContainer).order_by(
|
|
(DockerContainer.status == "running").desc(),
|
|
DockerContainer.name,
|
|
)
|
|
)).scalars())
|
|
hosts = await _host_map(db, {c.host_id for c in containers})
|
|
|
|
# Group by host so each container is clearly attributed to the box it
|
|
# runs on (names are only unique within a host now).
|
|
groups: dict[str, dict] = {}
|
|
for c in containers:
|
|
g = groups.get(c.host_id)
|
|
if g is None:
|
|
host = hosts.get(c.host_id)
|
|
g = groups[c.host_id] = {
|
|
"host": host,
|
|
"host_id": c.host_id,
|
|
"host_name": host.name if host else c.host_id,
|
|
"containers": [], "running": 0, "stopped": 0,
|
|
}
|
|
cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since)
|
|
g["containers"].append({
|
|
"container": c,
|
|
"ports": json.loads(c.ports_json) if c.ports_json else [],
|
|
"uptime": _human_uptime(c.started_at) if c.status == "running" else None,
|
|
"sparkline_cpu": _sparkline(cpu_hist),
|
|
"sparkline_mem": _sparkline(mem_hist),
|
|
})
|
|
if c.status == "running":
|
|
g["running"] += 1
|
|
else:
|
|
g["stopped"] += 1
|
|
|
|
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
|
|
running = sum(g["running"] for g in host_groups)
|
|
total = len(containers)
|
|
return await render_template(
|
|
"docker/rows.html",
|
|
host_groups=host_groups,
|
|
running=running,
|
|
stopped=total - running,
|
|
total=total,
|
|
range_key=range_key,
|
|
)
|
|
|
|
|
|
@docker_bp.get("/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def widget():
|
|
"""HTMX dashboard widget: container status overview, grouped by host."""
|
|
show_stopped = request.args.get("show_stopped", "no") == "yes"
|
|
widget_id = request.args.get("wid", "0")
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
all_containers = list((await db.execute(
|
|
select(DockerContainer).order_by(
|
|
(DockerContainer.status == "running").desc(),
|
|
DockerContainer.name,
|
|
)
|
|
)).scalars())
|
|
hosts = await _host_map(db, {c.host_id for c in all_containers})
|
|
|
|
running = [c for c in all_containers if c.status == "running"]
|
|
stopped = [c for c in all_containers if c.status != "running"]
|
|
display = all_containers if show_stopped else running
|
|
|
|
# Group for display so multi-host fleets read clearly; single-host stays flat.
|
|
groups: dict[str, dict] = {}
|
|
for c in display:
|
|
g = groups.setdefault(c.host_id, {
|
|
"host_id": c.host_id,
|
|
"host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
|
|
"containers": [],
|
|
})
|
|
g["containers"].append(c)
|
|
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
|
|
|
|
return await render_template(
|
|
"docker/widget.html",
|
|
host_groups=host_groups,
|
|
multi_host=len(host_groups) > 1,
|
|
running_count=len(running),
|
|
stopped_count=len(stopped),
|
|
show_stopped=show_stopped,
|
|
widget_id=widget_id,
|
|
)
|
|
|
|
|
|
@docker_bp.get("/widget/resources")
|
|
@require_role(UserRole.viewer)
|
|
async def widget_resources():
|
|
"""HTMX dashboard widget: CPU + memory usage for the busiest containers."""
|
|
limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
|
|
widget_id = request.args.get("wid", "0")
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
containers = list((await db.execute(
|
|
select(DockerContainer)
|
|
.where(DockerContainer.status == "running")
|
|
.order_by(DockerContainer.cpu_pct.desc().nullslast())
|
|
)).scalars())[:limit]
|
|
hosts = await _host_map(db, {c.host_id for c in containers})
|
|
|
|
rows_data = [
|
|
{"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
|
|
for c in containers
|
|
]
|
|
return await render_template(
|
|
"docker/widget_resources.html",
|
|
rows=rows_data,
|
|
multi_host=len({c.host_id for c in containers}) > 1,
|
|
widget_id=widget_id,
|
|
)
|
|
|
|
|
|
@docker_bp.get("/host/<host_id>")
|
|
@require_role(UserRole.viewer)
|
|
async def host_panel(host_id: str):
|
|
"""Per-host Docker fragment embedded on the core Hosts hub via HTMX.
|
|
|
|
Renders nothing when the host reports no containers, so hosts without Docker
|
|
don't carry an empty card.
|
|
"""
|
|
async with current_app.db_sessionmaker() as db:
|
|
containers = list((await db.execute(
|
|
select(DockerContainer)
|
|
.where(DockerContainer.host_id == host_id)
|
|
.order_by(
|
|
(DockerContainer.status == "running").desc(),
|
|
DockerContainer.name,
|
|
)
|
|
)).scalars())
|
|
|
|
if not containers:
|
|
return ""
|
|
running = sum(1 for c in containers if c.status == "running")
|
|
return await render_template(
|
|
"docker/host_panel.html",
|
|
containers=containers,
|
|
running=running,
|
|
stopped=len(containers) - running,
|
|
)
|