fix(maintenance): a late return, once back, stays back, and every reader agrees (#6175)

18032a4 re-checked a "not back in service" job every minute with no end,
so any later swarm wobble on the host (radarr's failed task during the
downloads move, say) flipped maintenance_failed back to 1: five false
firings and ten emails for Docker02 overnight on 10-19. And the overview's
finding never learned the rule at all, so it still said Docker02's
maintenance failed three days on.

The job now keeps came_back_at, set the first time its host is found back.
maintenance.still_failed(job) is the one answer: the figure, the overview,
the cadence page's run rows and the host panel all ask it. A failed update
or a timed-out job still flags its host until a job succeeds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/cadence.py

@@ -29,9 +29,6 @@ WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday",
 SETTLE_LIMIT = timedelta(minutes=30)
 # An agent heard from this recently is posting.
 POSTING_WITHIN = timedelta(minutes=3)
-# How a job failed for not coming back into service begins: its update (and
-# reboot) went through, so once the host is back nothing is left failed.
-NOT_BACK = "not back in service within"
 RUN_STATES = ("running", "done", "failed", "stopped")
 
 
@@ -132,7 +129,7 @@ def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str
     if back is None:
         return "next", None
     if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
-        return "fail", f"{NOT_BACK} {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
+        return "fail", f"{maintenance.NOT_BACK} {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
     return "settling", back
 
 
@@ -303,7 +300,9 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
     figure an alert rule watches. A host with no job yet reads 0, so a rule
     on it reports rather than sitting at "no data". A later job that
     succeeds puts it back to 0, and so does a host that failed only for not
-    coming back into service in time, once it is back (#6105)."""
+    coming back into service in time, once it is back (#6105). That return
+    is kept on the job (came_back_at), so a later wobble in the host's
+    services doesn't flag it again (#6175)."""
     from sqlalchemy import select
     from steward.models.hosts import Host
     from .models import HostAgentRegistration, HostMaintenanceJob
@@ -320,10 +319,11 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
         if host.name not in latest or job.finished_at > latest[host.name][1].finished_at:
             latest[host.name] = (host, job)
     for name, (host, job) in latest.items():
-        failed = job.state != "done"
-        if failed and (job.detail or "").startswith(NOT_BACK):
+        if (job.state == "failed" and job.came_back_at is None
+                and (job.detail or "").startswith(maintenance.NOT_BACK)):
             reg = (await session.execute(select(HostAgentRegistration).where(
                 HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
-            failed = await _back(session, host, job, reg, now) is not None
-        out[name] = 1.0 if failed else 0.0
+            if await _back(session, host, job, reg, now) is None:
+                job.came_back_at = now
+        out[name] = 1.0 if maintenance.still_failed(job) else 0.0
     return out