feat(ansible): a swarm prune that keeps every image a swarm service is pinned to on every node (#6399)

The nightly prune ran `docker system prune -a`, which deletes every image no
local container uses, so each swarm node kept only its own tasks' images.
When the other node failed or was drained, its services had to pull first:
on 2026-10-25 Traefik took 11.5 min to start on Docker03.

maintenance/swarm_prune.yml (built in) prunes stopped containers, unused
networks, dangling images and build cache, then removes only the images that
no container uses and no swarm service is pinned to, wherever that service
runs. Old digests left by redeploys still go.

- A container's image is kept by its id and by the reference it was made
  from, since on the containerd store the id alone hasn't held a tag.
- A node that can't list the swarm's services removes no images and fails
  rather than pruning blind.
- Check mode lists what would go.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/ansible/bundled/maintenance/files/swarm_prune.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Steward's swarm prune (maintenance/swarm_prune.yml), run on each node.
+
+Removes the images on this node that no swarm service is pinned to and no
+container uses (by id, or by the reference it was made from), and keeps every image a service is pinned to even when its
+task runs on another node. A node then holds what it needs to take over the
+other's services without a pull: the registry may be behind the node that
+failed, and a digest it has since deleted can't be pulled at all.
+
+The node must be a swarm manager, the only kind that can list services. On
+one that isn't, or when Docker doesn't answer, it removes nothing and fails,
+so a prune never runs blind. With --dry-run it says what it would remove.
+Prints one JSON line: kept, removed, failed, freed_bytes, dry_run.
+
+Stdlib only; shipped beside the playbook in Steward's built-in source.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+
+DEADLINE_SECONDS = 300
+
+
+def docker(*args: str) -> str:
+    done = subprocess.run(["docker", *args], capture_output=True, text=True,
+                          timeout=DEADLINE_SECONDS)
+    if done.returncode != 0:
+        raise RuntimeError(f"docker {' '.join(args[:2])}: {done.stderr.strip()[-300:]}")
+    return done.stdout
+
+
+def short(repo: str) -> str:
+    """A repository the way Docker lists it locally: no docker.io/ or library/."""
+    for prefix in ("docker.io/", "index.docker.io/"):
+        if repo.startswith(prefix):
+            repo = repo[len(prefix):]
+    return repo[len("library/"):] if repo.startswith("library/") else repo
+
+
+def wanted(service_images) -> tuple[set, set]:
+    """(digests, "repo:tag" refs) the services are pinned to. A spec with a
+    digest is matched by it; one with a tag alone by that tag."""
+    digests, refs = set(), set()
+    for image in service_images:
+        ref, _, digest = (image or "").partition("@")
+        if digest:
+            digests.add(digest)
+        elif ref:
+            name = ref.rsplit("/", 1)[-1]
+            refs.add(short(ref if ":" in name else ref + ":latest"))
+    return digests, refs
+
+
+def removable(images: list, digests: set, refs: set, used: set) -> list:
+    """The images (`docker image inspect` rows) no container uses and no
+    service is pinned to."""
+    out = []
+    for im in images:
+        if im.get("Id") in used:
+            continue
+        held = {d.split("@", 1)[1] for d in im.get("RepoDigests") or [] if "@" in d}
+        tags = {short(t) for t in im.get("RepoTags") or []}
+        if held & digests or tags & refs:
+            continue
+        out.append(im)
+    return out
+
+
+def label(im: dict) -> str:
+    names = (im.get("RepoTags") or []) + (im.get("RepoDigests") or [])
+    return names[0] if names else im.get("Id", "")[7:19]
+
+
+def main(argv: list) -> int:
+    dry = "--dry-run" in argv
+    try:
+        ids = docker("service", "ls", "-q").split()
+        services = json.loads(docker("service", "inspect", *ids)) if ids else []
+        ids = docker("ps", "-aq", "--no-trunc").split()
+        containers = json.loads(docker("container", "inspect", *ids)) if ids else []
+        image_ids = sorted(set(docker("image", "ls", "-q", "--no-trunc").split()))
+        images = json.loads(docker("image", "inspect", *image_ids)) if image_ids else []
+    except (RuntimeError, ValueError, OSError, subprocess.TimeoutExpired) as exc:
+        print(json.dumps({"error": f"nothing removed: {exc}"}))
+        return 2
+    # A container's image is kept by its id and by the reference it was made
+    # from: on Docker's containerd store the id alone hasn't held a tag.
+    used = {c.get("Image") for c in containers}
+    digests, refs = wanted([s.get("Spec", {}).get("TaskTemplate", {}).get("ContainerSpec", {}).get("Image")
+                            for s in services]
+                           + [c.get("Config", {}).get("Image") for c in containers])
+    gone = removable(images, digests, refs, used)
+    removed, failed, freed = [], [], 0
+    for im in gone:
+        if not dry:
+            try:
+                docker("image", "rm", "-f", im["Id"])
+            except (RuntimeError, subprocess.TimeoutExpired) as exc:
+                failed.append({"image": label(im), "error": str(exc)[-200:]})
+                continue
+        removed.append(label(im))
+        freed += int(im.get("Size") or 0)
+    print(json.dumps({"kept": len(images) - len(gone), "removed": removed, "failed": failed,
+                      "freed_bytes": freed, "dry_run": dry}))
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

steward/ansible/bundled/maintenance/swarm_prune.yml

@@ -0,0 +1,43 @@
+---
+# description: Reclaim disk on swarm nodes but keep every image a swarm service is pinned to, so each node can take over the others' services without pulling.
+# steward:category: maintenance
+# steward:confirm: true
+# Run on the swarm's nodes; each must be a manager (only a manager can list
+# services). On each node:
+#   1. docker system prune: stopped containers, unused networks, dangling
+#      images, build cache (+ prune_volumes=true: unused anonymous volumes).
+#   2. files/swarm_prune.py: removes every other image no container uses and
+#      no swarm service is pinned to, wherever that service's task runs. A
+#      node keeps the images for the services running elsewhere, so it can
+#      start them at once when the other node fails or is drained.
+# A node that can't list the swarm's services removes no images and the run
+# fails, rather than pruning blind. Check mode lists what would go.
+- name: Swarm prune
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    prune_volumes: false
+  tasks:
+    - name: Prune stopped containers, unused networks, dangling images and build cache
+      ansible.builtin.command:
+        argv: >-
+          {{ ['docker', 'system', 'prune', '-f']
+             + (['--volumes'] if prune_volumes | bool else []) }}
+      register: system_prune
+      changed_when: "'Total reclaimed space: 0B' not in system_prune.stdout"
+
+    - name: Remove images no container uses and no swarm service is pinned to
+      ansible.builtin.script:
+        cmd: "swarm_prune.py{{ ' --dry-run' if ansible_check_mode else '' }}"
+        executable: python3
+      check_mode: false
+      register: image_prune
+      changed_when: >-
+        not ansible_check_mode and (image_prune.stdout | from_json).removed | default([]) | length > 0
+
+    - name: Report what went
+      ansible.builtin.debug:
+        msg: >-
+          {{ (system_prune.stdout_lines | default([]) | select | list)[-1:]
+             + [image_prune.stdout | from_json] }}

tests/core/test_swarm_prune.py

@@ -0,0 +1,118 @@
+"""The swarm prune keeps every image a swarm service is pinned to on every
+node, removes the rest no container uses, and removes nothing when it can't
+see the swarm."""
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+import yaml
+
+from steward.ansible.sources import discover_playbook_meta, discover_playbooks, get_sources
+
+BUILTIN = Path(get_sources({"sources": []})[0]["path"])
+
+
+@pytest.fixture
+def prune():
+    spec = importlib.util.spec_from_file_location(
+        "swarm_prune", BUILTIN / "maintenance" / "files" / "swarm_prune.py")
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def _image(id_, tags=(), digests=(), size=100):
+    return {"Id": id_, "RepoTags": list(tags), "RepoDigests": list(digests), "Size": size}
+
+
+def test_the_playbook_is_listed_and_asks_before_it_runs():
+    assert "maintenance/swarm_prune.yml" in discover_playbooks(str(BUILTIN))
+    content = (BUILTIN / "maintenance" / "swarm_prune.yml").read_text()
+    meta = discover_playbook_meta(content)
+    assert meta["confirm"] is True and meta["category"] == "maintenance" and meta["description"]
+    play = yaml.safe_load(content)[0]
+    script = next(t for t in play["tasks"] if "ansible.builtin.script" in t)
+    assert script["check_mode"] is False and "--dry-run" in script["ansible.builtin.script"]["cmd"]
+
+
+def test_services_are_matched_by_digest_or_else_by_tag(prune):
+    digests, refs = prune.wanted([
+        "registry.example/team/app:dev@sha256:aaa",
+        "docker.io/library/redis:7",
+        "registry.example:5000/tool",
+        None,
+    ])
+    assert digests == {"sha256:aaa"}
+    assert refs == {"redis:7", "registry.example:5000/tool:latest"}
+
+
+def test_an_image_a_service_or_a_container_needs_stays(prune):
+    images = [
+        _image("sha256:1", ["registry.example/team/app:dev"], ["registry.example/team/app@sha256:aaa"]),
+        _image("sha256:2", ["registry.example/team/app:dev-old"], ["registry.example/team/app@sha256:old"]),
+        _image("sha256:3", ["redis:7"], ["redis@sha256:rrr"]),
+        _image("sha256:4", ["tool:1"]),
+        _image("sha256:5", [], ["registry.example/team/app@sha256:bbb"]),
+    ]
+    gone = prune.removable(images, {"sha256:aaa"}, {"redis:7"}, used={"sha256:4"})
+    assert [im["Id"] for im in gone] == ["sha256:2", "sha256:5"]
+
+
+def _fake_docker(monkeypatch, prune, answers, removed):
+    def docker(*args):
+        if args[:2] == ("image", "rm"):
+            removed.append(args[-1])
+            return ""
+        key = " ".join(args[:2])
+        if key not in answers:
+            raise RuntimeError(f"docker {key}: unexpected")
+        answer = answers[key]
+        if isinstance(answer, Exception):
+            raise answer
+        return answer
+    monkeypatch.setattr(prune, "docker", docker)
+
+
+def _answers():
+    return {
+        "service ls": "s1 s2\n",
+        "service inspect": json.dumps([
+            {"Spec": {"TaskTemplate": {"ContainerSpec": {"Image": "registry.example/app:dev@sha256:aaa"}}}},
+            {"Spec": {"TaskTemplate": {"ContainerSpec": {"Image": "proxy:3@sha256:ppp"}}}}]),
+        "ps -aq": "c1\n",
+        "container inspect": json.dumps([{"Image": "sha256:9", "Config": {"Image": "local:1"}}]),
+        "image ls": "sha256:1\nsha256:2\nsha256:3\nsha256:9\n",
+        "image inspect": json.dumps([
+            _image("sha256:1", ["registry.example/app:dev"], ["registry.example/app@sha256:aaa"]),
+            _image("sha256:2", ["proxy:3"], ["proxy@sha256:ppp"]),
+            _image("sha256:3", ["registry.example/app:old"], ["registry.example/app@sha256:old"], size=500),
+            _image("sha256:9", ["local:1"]),
+        ]),
+    }
+
+
+def test_a_run_removes_only_what_nothing_needs_and_says_so(prune, monkeypatch, capsys):
+    removed: list = []
+    _fake_docker(monkeypatch, prune, _answers(), removed)
+    assert prune.main([]) == 0
+    out = json.loads(capsys.readouterr().out)
+    assert removed == ["sha256:3"]
+    assert out == {"kept": 3, "removed": ["registry.example/app:old"], "failed": [],
+                   "freed_bytes": 500, "dry_run": False}
+
+
+def test_a_dry_run_removes_nothing(prune, monkeypatch, capsys):
+    removed: list = []
+    _fake_docker(monkeypatch, prune, _answers(), removed)
+    assert prune.main(["--dry-run"]) == 0
+    assert removed == [] and json.loads(capsys.readouterr().out)["removed"] == ["registry.example/app:old"]
+
+
+def test_a_node_that_cant_list_the_swarm_removes_nothing(prune, monkeypatch, capsys):
+    removed: list = []
+    answers = _answers()
+    answers["service ls"] = RuntimeError("docker service ls: This node is not a swarm manager.")
+    _fake_docker(monkeypatch, prune, answers, removed)
+    assert prune.main([]) == 2
+    assert removed == [] and "nothing removed" in json.loads(capsys.readouterr().out)["error"]