feat(swarm): services short, nodes down and managers lost are alertable (#5788)

On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each
manager's swarm report the docker plugin now records three figures on
resource "swarm": swarm_services_short, swarm_nodes_down and
swarm_managers_unreachable. Drained nodes are left out of the last two,
since a drain is deliberate. The figures are recorded at most once every
30 s whichever manager reported, and are offered to alert rules.

Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents
fall back to the node's status.

A maintenance job on a swarm node also keeps a window over docker/swarm,
so its drain and reboot hold those alerts until the job ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/maintenance.py

@@ -188,21 +188,47 @@ def utcnow() -> datetime:
 
 async def _window(session, job, host_name: str, now: datetime) -> None:
     """Keep the job's maintenance window over the host until the job ends: one
-    scoped to this host's agent metrics, so an alert the job causes waits."""
+    scoped to this host's agent metrics, so an alert the job causes waits.
+    A swarm node's job keeps a second over the swarm-wide figures (#5788):
+    draining and rebooting a node takes it down and can leave a service
+    pinned to it short, which is the job, not news."""
     from steward.models.alerts import MaintenanceWindow
     if job.window_id:
         w = await session.get(MaintenanceWindow, job.window_id)
         if w is not None:
             w.end_at = max(w.start_at, window_end(job, now))
-        return
-    if job.created_by is None or job.state in FINISHED:
+    elif job.created_by is None or job.state in FINISHED:
         return                          # a window needs an owner; none for a finished job
-    w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
-                          end_at=window_end(job, now), scope_source="host_agent",
-                          scope_resource=host_name, created_by=job.created_by)
-    session.add(w)
-    await session.flush()
-    job.window_id = w.id
+    else:
+        w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
+                              end_at=window_end(job, now), scope_source="host_agent",
+                              scope_resource=host_name, created_by=job.created_by)
+        session.add(w)
+        await session.flush()
+        job.window_id = w.id
+    if job.node_id:
+        await _swarm_window(session, job, host_name, now)
+
+
+SWARM_SCOPE = ("docker", "swarm")       # the swarm-wide figures' source and resource
+
+
+async def _swarm_window(session, job, host_name: str, now: datetime) -> None:
+    """The swarm half of a swarm node's job window, found again by its name
+    and the job's start, so it needs no column of its own."""
+    from sqlalchemy import select
+    from steward.models.alerts import MaintenanceWindow
+    name = f"Maintenance: {host_name} (swarm)"[:128]
+    w = (await session.execute(select(MaintenanceWindow).where(
+        MaintenanceWindow.name == name, MaintenanceWindow.scope_source == SWARM_SCOPE[0],
+        MaintenanceWindow.scope_resource == SWARM_SCOPE[1],
+        MaintenanceWindow.start_at >= job.created_at))).scalars().first()
+    if w is not None:
+        w.end_at = max(w.start_at, window_end(job, now))
+    elif job.created_by is not None and job.state not in FINISHED:
+        session.add(MaintenanceWindow(name=name, start_at=now, end_at=window_end(job, now),
+                                      scope_source=SWARM_SCOPE[0], scope_resource=SWARM_SCOPE[1],
+                                      created_by=job.created_by))
 
 
 async def _host_name(session, host_id: str) -> str: