diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index c64a4e2..aca8d1f 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -67,5 +67,7 @@ def get_nav() -> list[dict]: def get_alert_metrics() -> dict[str, list[str]]: """What ingest.py records per container (resource "/"): restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 / - 0.0 from the container's HEALTHCHECK, only for containers that define one.""" - return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"]} + 0.0 from the container's HEALTHCHECK, only for containers that define one. + Per host (resource ""), what Docker keeps on its disk (#5304).""" + from .disk_view import DISK_METRICS + return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS]} diff --git a/plugins/docker/disk_view.py b/plugins/docker/disk_view.py new file mode 100644 index 0000000..141b739 --- /dev/null +++ b/plugins/docker/disk_view.py @@ -0,0 +1,82 @@ +# plugins/docker/disk_view.py +"""What Docker keeps on each host's disk, over time and by image repository +(#5304–#5306). Model-free, so it is unit-tested without a database. + +The agent reports `docker system df` as current state; ingest.py records its +totals as metrics too (DISK_METRICS, resource = the host's name), so each +host's growth has a history and an alert rule can watch it. +""" +from __future__ import annotations + +from datetime import datetime + +# metric name → the df summary field it records; "disk_total_bytes" is their sum. +DISK_PARTS = { + "disk_images_bytes": "images_size", + "disk_containers_bytes": "containers_size", + "disk_volumes_bytes": "volumes_size", + "disk_build_cache_bytes": "build_cache_size", +} +DISK_METRICS = [*DISK_PARTS, "disk_total_bytes", "disk_reclaimable_bytes"] + + +def disk_metrics(disk: dict) -> dict[str, float]: + """The metric samples one df report gives: each part, their total, and + what unused images hold.""" + out = {m: float(int(disk.get(f) or 0)) for m, f in DISK_PARTS.items()} + out["disk_total_bytes"] = sum(out.values()) + out["disk_reclaimable_bytes"] = float(int(disk.get("images_reclaimable") or 0)) + return out + + +def growth(points: list[tuple[datetime, float]]) -> dict | None: + """How much a total changed over the points given (oldest first): the + change, the first point's time and value. None with fewer than two.""" + if len(points) < 2: + return None + (first_at, first), (_, last) = points[0], points[-1] + return {"change": last - first, "since": first_at, "from": first, "to": last, + "pct": (last - first) / first * 100 if first else None} + + +def image_repo(repo_tag: str | None) -> tuple[str | None, str | None]: + """("ghcr.io/x/app", "1.2") for "ghcr.io/x/app:1.2"; a registry's port is + part of the repository ("reg:5000/app" has no tag); a digest reference + keeps its digest as the tag; (None, None) for an untagged image.""" + if not repo_tag or repo_tag == "" or repo_tag.startswith(""): + return None, None + if "@" in repo_tag: + repo, _, digest = repo_tag.partition("@") + return repo, digest[:19] + head, sep, tag = repo_tag.rpartition(":") + if not sep or "/" in tag: + return repo_tag, None + return head, tag + + +def group_images(images: list[dict]) -> list[dict]: + """Images folded under their repository, the most space first. + + Each image: {repo_tag, image_id, size, shared_size, containers}. A group's + `own` counts each version's size less the layers it shares with other + images, so a repository is not charged twice for one base layer; + `reclaimable` is that, for the versions no container uses. Untagged images + form one group of their own (repo None).""" + groups: dict[str | None, dict] = {} + for im in images: + repo, tag = image_repo(im.get("repo_tag")) + g = groups.setdefault(repo, {"repo": repo, "versions": [], "own": 0, "size": 0, + "reclaimable": 0, "containers": 0}) + own = max(int(im.get("size") or 0) - int(im.get("shared_size") or 0), 0) + used = int(im.get("containers") or 0) + g["versions"].append({"tag": tag, "image_id": im.get("image_id"), "size": int(im.get("size") or 0), + "shared": int(im.get("shared_size") or 0), "own": own, "containers": used}) + g["own"] += own + g["size"] = max(g["size"], int(im.get("size") or 0)) + g["containers"] += used + if not used: + g["reclaimable"] += own + for g in groups.values(): + g["versions"].sort(key=lambda v: (v["containers"] == 0, -v["size"])) + g["unused"] = sum(1 for v in g["versions"] if not v["containers"]) + return sorted(groups.values(), key=lambda g: (g["repo"] is None, -g["own"], g["repo"] or "")) diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py index d493f21..46bcc99 100644 --- a/plugins/docker/ingest.py +++ b/plugins/docker/ingest.py @@ -240,6 +240,14 @@ async def _persist_disk(session, host, disk: dict) -> None: summary.containers_count = int(disk.get("containers_count") or 0) summary.volumes_count = int(disk.get("volumes_count") or 0) summary.updated_at = now + # The totals as metrics too (#5304): a history of each host's growth, and + # something an alert rule can watch. Resource: the host's name. + from steward.core.alerts import MetricBatch + from .disk_view import disk_metrics + batch = MetricBatch() + for metric, value in disk_metrics(disk).items(): + batch.add(source_module="docker", resource_name=host.name, metric_name=metric, value=value) + await batch.record(session) images = disk.get("images") or [] seen: set[str] = set() diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index a9e1050..8c2edfe 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -110,11 +110,15 @@ async def index(): has_disk = (await db.execute( select(DockerDiskUsage.host_id).limit(1))).first() is not None choices = await _filter_choices(db) + # One line per host under the tables: what Docker keeps on its disk and + # how much that grew in a week (#5306); the Disk page has the rest. + disk = await docker_svc.disk_hosts( + db, datetime.now(timezone.utc) - timedelta(days=7), points=14) if has_disk else [] return await render_template( "docker/index.html", poll_interval=poll_interval, has_swarm=has_swarm, - has_disk=has_disk, + has_disk=has_disk, disk=disk, f=_filters(request.args), choices=choices, states=DOCKER_STATES, no_stack=NO_STACK, ) @@ -627,7 +631,9 @@ IMAGES_REPORTED = 50 @docker_bp.get("/disk") @require_role(UserRole.viewer) async def disk(): - """Image/disk usage page: reclaimable space, per-image sizes, stopped count. + """What Docker keeps on each host's disk (#5304, #5305): the hosts side by + side with how much each grew over the selected range (the /disk/hosts + fragment), then each host's parts, cleanup and images by repository. Admins can reclaim space per host via the prune buttons, which fire the audited bundled prune playbook through Ansible (m78) — the collection agent @@ -635,6 +641,7 @@ async def disk(): """ from steward.core.capabilities import has_capability from steward.models.ansible_inventory import AnsibleTarget + from .disk_view import group_images async with current_app.db_sessionmaker() as db: summaries = list((await db.execute(select(DockerDiskUsage))).scalars()) @@ -664,43 +671,48 @@ async def disk(): images_by_host: dict[str, list] = {} for im in images: images_by_host.setdefault(im.host_id, []).append({ - "repo_tag": None if im.repo_tag in ("", "", None) else im.repo_tag, - "image_id": im.image_id, "size": human_bytes(im.size), - "shared": human_bytes(im.shared_size), "containers": im.containers, - "reclaimable": im.containers == 0, - }) + "repo_tag": im.repo_tag, "image_id": im.image_id, "size": im.size, + "shared_size": im.shared_size, "containers": im.containers}) host_groups = [ { "host_id": s.host_id, "host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id, "as_of": s.updated_at, - "summary": { - "images_total": s.images_total, "images_active": s.images_active, - "images_size": human_bytes(s.images_size), - "images_reclaimable": human_bytes(s.images_reclaimable), - "layers_size": human_bytes(s.layers_size), - "containers_count": s.containers_count, - "containers_size": human_bytes(s.containers_size), - "volumes_count": s.volumes_count, - "volumes_size": human_bytes(s.volumes_size), - "build_cache_size": human_bytes(s.build_cache_size), - }, + "summary": s, + "total": s.images_size + s.containers_size + s.volumes_size + s.build_cache_size, "stopped": stopped_by_host.get(s.host_id, 0), - "images": images_by_host.get(s.host_id, []), + "repos": group_images(images_by_host.get(s.host_id, [])), + "images_listed": len(images_by_host.get(s.host_id, [])), # The agent reports the largest IMAGES_REPORTED images (agent._disk_images). "images_capped": len(images_by_host.get(s.host_id, [])) >= IMAGES_REPORTED, "has_target": s.host_id in target_by_host, } for s in summaries ] - host_groups.sort(key=lambda g: g["host_name"].lower()) + host_groups.sort(key=lambda g: (-g["total"], g["host_name"].lower())) return await render_template( "docker/disk.html", host_groups=host_groups, ansible_available=has_capability("ansible.run_playbook"), + current_range=request.args.get("range", "7d"), ) +@docker_bp.get("/disk/hosts") +@require_role(UserRole.viewer) +async def disk_hosts_fragment(): + """HTMX fragment: every Docker host's disk side by side, with how much its + total grew over the selected range and a line of it (#5304).""" + from steward.core.status import sparkline_svg + since, range_key = parse_range(request.args.get("range") or "7d") + async with current_app.db_sessionmaker() as db: + rows = await docker_svc.disk_hosts(db, since) + for r in rows: + r["spark"] = sparkline_svg([v for _, v in r["points"]], width=120, height=24, + stroke="currentColor") + return await render_template("docker/_disk_hosts.html", rows=rows, range_key=range_key) + + @docker_bp.post("/disk//prune") @require_role(UserRole.admin) async def disk_prune(host_id: str): diff --git a/plugins/docker/service.py b/plugins/docker/service.py index fd68276..aa63baf 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -683,3 +683,25 @@ def stale_after_seconds(config) -> int: """How old a container's last report may be before the page calls it stale: the host agent's own staleness window, since the agent reports it.""" return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180)) + + +async def disk_hosts(db, since: datetime, *, points: int = 60) -> list[dict]: + """Each Docker host's disk as `docker system df` last said, most space + first, with its recorded total since `since` (#5304): the points, oldest + first, and the growth over them (disk_view.growth; None until there are + two). Hosts are named by their Steward host.""" + from steward.core import metric_series + from .disk_view import DISK_PARTS, growth + from .models import DockerDiskUsage + summaries = list((await db.execute(select(DockerDiskUsage))).scalars()) + hosts = await host_map(db, {s.host_id for s in summaries}) + out = [] + for s in summaries: + name = hosts[s.host_id].name if s.host_id in hosts else s.host_id + parts = {m: int(getattr(s, f) or 0) for m, f in DISK_PARTS.items()} + series = await metric_series.points(db, "docker", name, "disk_total_bytes", since, target=points) + out.append({"host_id": s.host_id, "host_name": name, "as_of": s.updated_at, + "total": sum(parts.values()), "parts": parts, + "reclaimable": int(s.images_reclaimable or 0), "points": series, + "growth": growth(series)}) + return sorted(out, key=lambda h: (-h["total"], h["host_name"].lower())) diff --git a/plugins/docker/templates/docker/_disk_hosts.html b/plugins/docker/templates/docker/_disk_hosts.html new file mode 100644 index 0000000..9513a7b --- /dev/null +++ b/plugins/docker/templates/docker/_disk_hosts.html @@ -0,0 +1,37 @@ +{# docker/_disk_hosts.html — every Docker host's disk side by side (#5304): + what it uses now, how much that grew over the selected range, and a line + of it; the parts after. A host is linked to its own section below. #} +{% if rows %} + + + + + + + + + + + {% for r in rows %} + {% set g = r.growth %} + + + + + + + + + + + + {% endfor %} + +
HostDocker usesChange ({{ range_key }})Over {{ range_key }}ImagesVolumesContainer filesBuild cacheReclaimable
{{ r.host_name }}{{ r.total|bytes }} + {%- if g is none %}— + {%- else %}{{ "+" if g.change > 0 else ("−" if g.change < 0 else "±") }}{{ (g.change|abs)|bytes }}{% endif -%} + {{ r.spark|safe }}{{ r.parts.disk_images_bytes|bytes }}{{ r.parts.disk_volumes_bytes|bytes }}{{ r.parts.disk_containers_bytes|bytes }}{{ r.parts.disk_build_cache_bytes|bytes }}{% if r.reclaimable %}{{ r.reclaimable|bytes }}{% else %}—{% endif %}
+{% else %} +

No host has reported its Docker disk yet.

+{% endif %} diff --git a/plugins/docker/templates/docker/disk.html b/plugins/docker/templates/docker/disk.html index 55aeb14..42031f2 100644 --- a/plugins/docker/templates/docker/disk.html +++ b/plugins/docker/templates/docker/disk.html @@ -1,127 +1,124 @@ -{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #} +{# docker/disk.html — what Docker keeps on each host's disk (#5304, #5305): + the hosts side by side with how much each grew, then each host's parts, + its cleanup actions, and its images folded under their repository. #} {% extends "base.html" %} -{% from "_macros.html" import crumbs %} +{% from "_macros.html" import crumbs, page_header, stat_tile, status_badge %} {% block title %}Disk — Docker — Steward{% endblock %} -{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Disk", "")]) }}{% endblock %} {% block content %} -

Image & disk usage

-

- What Docker is keeping on each host's disk, from docker system df. - Admins can clean up per host below; each action runs an audited Ansible playbook on that host. -

+{% call page_header("Docker disk", "What Docker keeps on each host's disk, from docker system df, and how it has grown.") %} + {% include "_time_range.html" %} +{% endcall %} {% if host_groups %} -{% for g in host_groups %} -
-
{{ g.host_name }} - {% if g.as_of %}· as of {{ g.as_of|ago }}{% endif %}
+
+

By host most space first

+
+

Loading…

+
+
- {# ── Summary stats ────────────────────────────────────────────────────── #} -
-
-
Reclaimable
- {{ g.summary.images_reclaimable }} -
held by images no container uses
-
-
-
Images
- {{ g.summary.images_size }} -
{{ g.summary.images_active }}/{{ g.summary.images_total }} in use
-
-
-
Stopped
- {{ g.stopped }} -
containers not running
-
-
-
Container files
- {{ g.summary.containers_size }} -
written by {{ g.summary.containers_count }} containers on top of their images
-
-
-
Volumes
- {{ g.summary.volumes_size }} -
{{ g.summary.volumes_count }} volumes; never pruned here
-
-
-
Build cache
- {{ g.summary.build_cache_size }} -
left by docker build; safe to clear
-
+{% for g in host_groups %} +{% set s = g.summary %} +
+

+ {{ g.host_name }} + {{ g.total|bytes }} + {% if g.as_of %}as of {{ g.as_of|ago }}{% endif %} +

+ +
+ {{ stat_tile("Reclaimable", s.images_reclaimable|bytes, state="warn" if s.images_reclaimable else "ok", hint="held by images no container uses") }} + {{ stat_tile("Images", s.images_size|bytes, hint=s.images_active ~ " of " ~ s.images_total ~ " in use") }} + {{ stat_tile("Container files", s.containers_size|bytes, hint="written by " ~ s.containers_count ~ " containers on top of their images") }} + {{ stat_tile("Volumes", s.volumes_size|bytes, hint=s.volumes_count ~ " volumes; never pruned here") }} + {{ stat_tile("Build cache", s.build_cache_size|bytes, hint="left by docker build; safe to clear") }} + {{ stat_tile("Stopped", g.stopped, state="dim" if not g.stopped else none, hint="containers not running") }}
{# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #} {% if session.user_role == 'admin' %} -
+
{% if ansible_available and g.has_target %} -
-
{{ csrf_field() }} - - -
-
{{ csrf_field() }} - - -
-
{{ csrf_field() }} - - -
-
-
- Reclaimed space appears on the next agent sample. -
+
{{ csrf_field() }} + + +
+
{{ csrf_field() }} + + +
+
{{ csrf_field() }} + + +
+ Each runs an audited Ansible playbook on {{ g.host_name }}; reclaimed space shows on the next agent sample. {% elif not ansible_available %} -
Cleanup needs the Ansible runner (currently unavailable).
+ Cleanup needs the Ansible runner (currently unavailable). {% else %} -
- Link an Ansible target to this host to enable prune actions. -
+ Link an Ansible target to this host to enable prune actions. {% endif %}
{% endif %} - {# ── Per-image table ──────────────────────────────────────────────────── #} -
- {% if g.images_capped %}The {{ g.images|length }} largest of {{ g.summary.images_total }} images, largest first. - {% else %}Every image, largest first.{% endif %} -
-
- - - - - - - - {% for im in g.images %} + {# ── Images by repository, versions folded under it (#5305) ─────────── #} +
+

Images by repository + {% if g.images_capped %}the {{ g.images_listed }} largest of {{ s.images_total }} images{% else %}{{ g.images_listed }} images{% endif %}

+ {% if g.repos %} +
ImageSizeShared with other imagesUsed by
+ + + + + + + + {% for r in g.repos %} + - - - - + + + + - {% else %} - - {% endfor %} + {% endfor %}
RepositoryVersionsOwn sizeUsed byReclaimable
- {% if im.repo_tag %}{{ im.repo_tag }}{% else %}untagged {{ im.image_id }}{% endif %} - {% if im.reclaimable %}reclaimable{% endif %} + +
+ {% if r.repo %}{{ r.repo }}{% else %}untagged{% endif %} + + + + {% for v in r.versions %} + + + + + + + + {% endfor %} + +
TagIDSizeSharedUsed by
{{ v.tag or "—" }}{{ (v.image_id or "")|replace("sha256:", "")|truncate(12, true, "") }}{{ v.size|bytes }}{{ v.shared|bytes }}{% if v.containers %}{{ v.containers }} container{{ "s" if v.containers != 1 }}{% else %}nothing{% endif %}
+
{{ im.size }}{{ im.shared }}{% if im.containers %}{{ im.containers }} container{{ "s" if im.containers != 1 }}{% else %}nothing{% endif %}{{ r.versions|length }}{% if r.unused %} ({{ r.unused }} unused){% endif %}{{ r.own|bytes }}{% if r.containers %}{{ r.containers }} container{{ "s" if r.containers != 1 }}{% else %}nothing{% endif %}{% if r.reclaimable %}{{ r.reclaimable|bytes }}{% else %}—{% endif %}
No images reported.
-
-
+ {% else %} +

No images reported.

+ {% endif %} +
+ {% endfor %} {% else %} -
-
- No disk usage reported yet. The host agent reports image/disk usage from - docker system df on hosts running Docker. -
+
+ No disk usage reported yet. The host agent reports what Docker keeps on disk, + from docker system df, on hosts running Docker.
{% endif %} {% endblock %} diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html index 1df9229..b57169e 100644 --- a/plugins/docker/templates/docker/index.html +++ b/plugins/docker/templates/docker/index.html @@ -41,6 +41,19 @@ hx-swap="innerHTML">

Loading…

+{% if disk %} +{# What Docker keeps on each host's disk, and its growth over a week (#5306). #} +

+ Disk + {% for d in disk %} + {{ d.host_name }} + {{ d.total|bytes }} + {%- if d.growth and d.growth.change %} {{ "+" if d.growth.change > 0 else "−" }}{{ (d.growth.change|abs)|bytes }} in 7d{% endif -%} + {%- if d.reclaimable %} · {{ d.reclaimable|bytes }} reclaimable{% endif %} + {% endfor %} +

+{% endif %}