fix(maintenance): a host that came back late stops reading as failed once it is back (#6105)

Docker02's Sunday job updated and rebooted, then failed only because
portal_app was short of its replica past the 30-minute settle. Its
maintenance_failed figure stayed 1, and the alert kept firing, until a
later job succeeds. A job failed for not coming back into service is
checked again each minute and reads 0 once the host is back; a failed
or timed-out update still reads 1 until a job succeeds.

Also: compose.deploy.yml's syslog comment still said devices are told
apart by address under Swarm (stale since #6096).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/cadence.py

@@ -29,6 +29,9 @@ WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday",
 SETTLE_LIMIT = timedelta(minutes=30)
 # An agent heard from this recently is posting.
 POSTING_WITHIN = timedelta(minutes=3)
+# How a job failed for not coming back into service begins: its update (and
+# reboot) went through, so once the host is back nothing is left failed.
+NOT_BACK = "not back in service within"
 RUN_STATES = ("running", "done", "failed", "stopped")
 
 
@@ -129,7 +132,7 @@ def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str
     if back is None:
         return "next", None
     if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
-        return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
+        return "fail", f"{NOT_BACK} {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
     return "settling", back
 
 
@@ -299,7 +302,8 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
     finished maintenance job failed or timed out, for the maintenance_failed
     figure an alert rule watches. A host with no job yet reads 0, so a rule
     on it reports rather than sitting at "no data". A later job that
-    succeeds puts it back to 0."""
+    succeeds puts it back to 0, and so does a host that failed only for not
+    coming back into service in time, once it is back (#6105)."""
     from sqlalchemy import select
     from steward.models.hosts import Host
     from .models import HostAgentRegistration, HostMaintenanceJob
@@ -307,13 +311,19 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
         select(Host.name).join(HostAgentRegistration, HostAgentRegistration.host_id == Host.id))).scalars()
     out = {name: 0.0 for name in agents}
     rows = (await session.execute(
-        select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
+        select(Host, HostMaintenanceJob)
         .join(Host, Host.id == HostMaintenanceJob.host_id)
         .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
                HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all()
     latest: dict = {}
-    for name, state, finished in rows:
-        if name not in latest or finished > latest[name][1]:
-            latest[name] = (state, finished)
-    out.update({name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()})
+    for host, job in rows:
+        if host.name not in latest or job.finished_at > latest[host.name][1].finished_at:
+            latest[host.name] = (host, job)
+    for name, (host, job) in latest.items():
+        failed = job.state != "done"
+        if failed and (job.detail or "").startswith(NOT_BACK):
+            reg = (await session.execute(select(HostAgentRegistration).where(
+                HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
+            failed = await _back(session, host, job, reg, now) is not None
+        out[name] = 1.0 if failed else 0.0
     return out