fix(maintenance): a host that came back late stops reading as failed once it is back (#6105)

Docker02's Sunday job updated and rebooted, then failed only because
portal_app was short of its replica past the 30-minute settle. Its
maintenance_failed figure stayed 1, and the alert kept firing, until a
later job succeeds. A job failed for not coming back into service is
checked again each minute and reads 0 once the host is back; a failed
or timed-out update still reads 1 until a job succeeds.

Also: compose.deploy.yml's syslog comment still said devices are told
apart by address under Swarm (stale since #6096).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

compose.deploy.yml

@@ -18,9 +18,10 @@ services:
       - "5000:5000"
       # Syslog (plugins/syslog): devices send here. Steward runs unprivileged,
       # so it listens above 1024; publish 514 to it instead ("514:5514/udp")
-      # if a device can't choose its port. Under Docker Swarm use the long form
-      # with `mode: host`: Swarm's default ingress publishing hands every packet
-      # over from one internal address, and devices are told apart by theirs.
+      # if a device can't choose its port. Under Docker Swarm's ingress every
+      # device arrives from one internal address and is told apart by the name
+      # its lines carry; the long form with `mode: host` keeps each device's
+      # own address but ties Steward to that node.
       - "5514:5514/udp"
       - "5514:5514/tcp"
     networks:

plugins/host_agent/cadence.py

@@ -29,6 +29,9 @@ WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday",
 SETTLE_LIMIT = timedelta(minutes=30)
 # An agent heard from this recently is posting.
 POSTING_WITHIN = timedelta(minutes=3)
+# How a job failed for not coming back into service begins: its update (and
+# reboot) went through, so once the host is back nothing is left failed.
+NOT_BACK = "not back in service within"
 RUN_STATES = ("running", "done", "failed", "stopped")
 
 
@@ -129,7 +132,7 @@ def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str
     if back is None:
         return "next", None
     if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
-        return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
+        return "fail", f"{NOT_BACK} {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
     return "settling", back
 
 
@@ -299,7 +302,8 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
     finished maintenance job failed or timed out, for the maintenance_failed
     figure an alert rule watches. A host with no job yet reads 0, so a rule
     on it reports rather than sitting at "no data". A later job that
-    succeeds puts it back to 0."""
+    succeeds puts it back to 0, and so does a host that failed only for not
+    coming back into service in time, once it is back (#6105)."""
     from sqlalchemy import select
     from steward.models.hosts import Host
     from .models import HostAgentRegistration, HostMaintenanceJob
@@ -307,13 +311,19 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
         select(Host.name).join(HostAgentRegistration, HostAgentRegistration.host_id == Host.id))).scalars()
     out = {name: 0.0 for name in agents}
     rows = (await session.execute(
-        select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
+        select(Host, HostMaintenanceJob)
         .join(Host, Host.id == HostMaintenanceJob.host_id)
         .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
                HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all()
     latest: dict = {}
-    for name, state, finished in rows:
-        if name not in latest or finished > latest[name][1]:
-            latest[name] = (state, finished)
-    out.update({name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()})
+    for host, job in rows:
+        if host.name not in latest or job.finished_at > latest[host.name][1].finished_at:
+            latest[host.name] = (host, job)
+    for name, (host, job) in latest.items():
+        failed = job.state != "done"
+        if failed and (job.detail or "").startswith(NOT_BACK):
+            reg = (await session.execute(select(HostAgentRegistration).where(
+                HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
+            failed = await _back(session, host, job, reg, now) is not None
+        out[name] = 1.0 if failed else 0.0
     return out

steward/core/formatting.py

@@ -356,7 +356,7 @@ METRICS: dict[str, tuple[str, str]] = {
     "updates_pending": ("Updates waiting", "packages the host's package manager would upgrade, checked hourly"),
     "updates_security": ("Security updates waiting", "of the updates waiting, those from a security source"),
     "reboot_required": ("Reboot required", "1 when the host needs a reboot to finish an update, 0 when not"),
-    "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed, timed out or didn't come back into service"),
+    "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed or timed out, or the host didn't come back into service and still isn't"),
     "temp_c_max": ("Hottest sensor", "the host's hottest real thermometer; inputs with nothing wired to them are left out"),
     "temp_c": ("Sensor temperature", "one temperature sensor (resource host:temp:label)"),
     "net_rx_bps": ("Network in", "bytes per second received, by the whole host or one interface"),

tests/integration/test_host_maintenance_cadence.py

@@ -134,3 +134,45 @@ def test_hosts_go_one_at_a_time_and_a_failure_stops_the_run():
         assert flags[seed["hosts"]["b"].name] == 1.0 and flags[seed["hosts"]["a"].name] == 0.0
 
     asyncio.run(_go())
+
+
+def test_a_host_that_came_back_late_reads_failed_only_until_it_is_back():
+    """#6105: a job whose update went through, failed only for not coming back
+    into service within the limit, stops flagging the host once it is back."""
+    app = _app()
+
+    async def _go():
+        from sqlalchemy import text
+        seed = await _seed(app)
+        cadence, models = _ha("cadence"), _ha("models")
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                plan = await s.get(models.HostMaintenancePlan, seed["plan_id"])
+                await cadence.start_run(s, plan, datetime.now(timezone.utc), manual=True)
+        await _tick(app)
+        ta, a = seed["tokens"]["a"], seed["hosts"]["a"]
+        (step,) = await _post(app, ta)
+        await _post(app, ta, [{"kind": "maintenance", "id": step["id"], "action": "update",
+                               "state": "done", "ok": True, "packages": 2, "reboot_required": False}])
+
+        async def _sql(sql):
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await s.execute(text(sql), {"h": a.id, "p": seed["plan_id"]})
+
+        # The agent goes quiet after its update, and stays so past the limit.
+        await _sql("UPDATE host_agent_registrations SET last_seen_at = now() - interval '1 hour' "
+                   "WHERE host_id = :h")
+        await _tick(app)
+        await _sql("UPDATE host_maintenance_runs SET settle_since = now() - interval '31 minutes' "
+                   "WHERE plan_id = :p")
+        flags = await _tick(app)
+        state, _, detail = await _run(app, seed["plan_id"])
+        assert state == "failed" and "not back in service within 30 min" in detail
+        assert flags[a.name] == 1.0
+
+        # Posting again, it is back in service: nothing is left failed.
+        await _post(app, ta)
+        assert (await _tick(app))[a.name] == 0.0
+
+    asyncio.run(_go())