diff --git a/compose.deploy.yml b/compose.deploy.yml index 2af7156..be745e7 100644 --- a/compose.deploy.yml +++ b/compose.deploy.yml @@ -18,9 +18,10 @@ services: - "5000:5000" # Syslog (plugins/syslog): devices send here. Steward runs unprivileged, # so it listens above 1024; publish 514 to it instead ("514:5514/udp") - # if a device can't choose its port. Under Docker Swarm use the long form - # with `mode: host`: Swarm's default ingress publishing hands every packet - # over from one internal address, and devices are told apart by theirs. + # if a device can't choose its port. Under Docker Swarm's ingress every + # device arrives from one internal address and is told apart by the name + # its lines carry; the long form with `mode: host` keeps each device's + # own address but ties Steward to that node. - "5514:5514/udp" - "5514:5514/tcp" networks: diff --git a/plugins/host_agent/cadence.py b/plugins/host_agent/cadence.py index d5aa0f9..5d900e7 100644 --- a/plugins/host_agent/cadence.py +++ b/plugins/host_agent/cadence.py @@ -29,6 +29,9 @@ WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", SETTLE_LIMIT = timedelta(minutes=30) # An agent heard from this recently is posting. POSTING_WITHIN = timedelta(minutes=3) +# How a job failed for not coming back into service begins: its update (and +# reboot) went through, so once the host is back nothing is left failed. +NOT_BACK = "not back in service within" RUN_STATES = ("running", "done", "failed", "stopped") @@ -129,7 +132,7 @@ def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str if back is None: return "next", None if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT: - return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}" + return "fail", f"{NOT_BACK} {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}" return "settling", back @@ -299,7 +302,8 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]: finished maintenance job failed or timed out, for the maintenance_failed figure an alert rule watches. A host with no job yet reads 0, so a rule on it reports rather than sitting at "no data". A later job that - succeeds puts it back to 0.""" + succeeds puts it back to 0, and so does a host that failed only for not + coming back into service in time, once it is back (#6105).""" from sqlalchemy import select from steward.models.hosts import Host from .models import HostAgentRegistration, HostMaintenanceJob @@ -307,13 +311,19 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]: select(Host.name).join(HostAgentRegistration, HostAgentRegistration.host_id == Host.id))).scalars() out = {name: 0.0 for name in agents} rows = (await session.execute( - select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at) + select(Host, HostMaintenanceJob) .join(Host, Host.id == HostMaintenanceJob.host_id) .where(HostMaintenanceJob.state.in_(maintenance.FINISHED), HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all() latest: dict = {} - for name, state, finished in rows: - if name not in latest or finished > latest[name][1]: - latest[name] = (state, finished) - out.update({name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}) + for host, job in rows: + if host.name not in latest or job.finished_at > latest[host.name][1].finished_at: + latest[host.name] = (host, job) + for name, (host, job) in latest.items(): + failed = job.state != "done" + if failed and (job.detail or "").startswith(NOT_BACK): + reg = (await session.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host.id))).scalar_one_or_none() + failed = await _back(session, host, job, reg, now) is not None + out[name] = 1.0 if failed else 0.0 return out diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 26b4415..a9a9023 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -356,7 +356,7 @@ METRICS: dict[str, tuple[str, str]] = { "updates_pending": ("Updates waiting", "packages the host's package manager would upgrade, checked hourly"), "updates_security": ("Security updates waiting", "of the updates waiting, those from a security source"), "reboot_required": ("Reboot required", "1 when the host needs a reboot to finish an update, 0 when not"), - "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed, timed out or didn't come back into service"), + "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed or timed out, or the host didn't come back into service and still isn't"), "temp_c_max": ("Hottest sensor", "the host's hottest real thermometer; inputs with nothing wired to them are left out"), "temp_c": ("Sensor temperature", "one temperature sensor (resource host:temp:label)"), "net_rx_bps": ("Network in", "bytes per second received, by the whole host or one interface"), diff --git a/tests/integration/test_host_maintenance_cadence.py b/tests/integration/test_host_maintenance_cadence.py index b6bb7a4..32956ed 100644 --- a/tests/integration/test_host_maintenance_cadence.py +++ b/tests/integration/test_host_maintenance_cadence.py @@ -134,3 +134,45 @@ def test_hosts_go_one_at_a_time_and_a_failure_stops_the_run(): assert flags[seed["hosts"]["b"].name] == 1.0 and flags[seed["hosts"]["a"].name] == 0.0 asyncio.run(_go()) + + +def test_a_host_that_came_back_late_reads_failed_only_until_it_is_back(): + """#6105: a job whose update went through, failed only for not coming back + into service within the limit, stops flagging the host once it is back.""" + app = _app() + + async def _go(): + from sqlalchemy import text + seed = await _seed(app) + cadence, models = _ha("cadence"), _ha("models") + async with app.db_sessionmaker() as s: + async with s.begin(): + plan = await s.get(models.HostMaintenancePlan, seed["plan_id"]) + await cadence.start_run(s, plan, datetime.now(timezone.utc), manual=True) + await _tick(app) + ta, a = seed["tokens"]["a"], seed["hosts"]["a"] + (step,) = await _post(app, ta) + await _post(app, ta, [{"kind": "maintenance", "id": step["id"], "action": "update", + "state": "done", "ok": True, "packages": 2, "reboot_required": False}]) + + async def _sql(sql): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text(sql), {"h": a.id, "p": seed["plan_id"]}) + + # The agent goes quiet after its update, and stays so past the limit. + await _sql("UPDATE host_agent_registrations SET last_seen_at = now() - interval '1 hour' " + "WHERE host_id = :h") + await _tick(app) + await _sql("UPDATE host_maintenance_runs SET settle_since = now() - interval '31 minutes' " + "WHERE plan_id = :p") + flags = await _tick(app) + state, _, detail = await _run(app, seed["plan_id"]) + assert state == "failed" and "not back in service within 30 min" in detail + assert flags[a.name] == 1.0 + + # Posting again, it is back in service: nothing is left failed. + await _post(app, ta) + assert (await _tick(app))[a.name] == 0.0 + + asyncio.run(_go())