feat(host_agent): a reboot reads as a reboot, and lost contact as its span (#5756)

#5756 "The agent says when its host rebooted and when it lost contact, so
downtime is recorded as what it was". Agent 1.14.0 posts events with the
next post that gets through: started (with the host's boot time), stopping
(on SIGTERM, with whether systemd says the host is going down) and
lost_contact (from its first unanswered post to the one carrying it).

Steward keeps them as the host's downtime (host_downtime): reboot,
agent_stop, unclean (a boot after the agent was last heard that nobody
announced) or lost_contact. Retried posts apply once.

An announced stop explains the silence for reboot_grace_seconds (Settings
-> Plugins -> host_agent, 10 minutes): seconds_since_seen reads 0 until
then, and the host's other rules go quiet with it (agents_away beside
#5721's registry). Past the grace, agent offline fires.

The host's Agent card lists the last 7 days of downtime. Rows are pruned
at data.retention_days.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_host_page.py

@@ -208,3 +208,30 @@ def test_sort_hosts_puts_missing_values_last_either_way():
     assert names("cpu", False) == ["alpha", "bravo", "charlie"]   # no reading: last both ways
     assert names("name", True) == ["charlie", "bravo", "alpha"]
     assert names("status", False)[0] == "charlie"                  # down first
+
+
+async def test_the_agent_card_says_what_happened_while_the_host_was_away(app):
+    """#5756: a reboot reads as a reboot with its length, one still going says so."""
+    from datetime import datetime, timedelta, timezone
+    from tests._templates import render_plugin_template
+    now = datetime.now(timezone.utc)
+    host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
+    reg = SimpleNamespace(agent_version="1.14.0", last_seen_at=now)
+    downtime = [
+        {"kind": "reboot", "label": "Rebooted", "started_at": now - timedelta(minutes=5),
+         "ended_at": None, "ongoing": True, "seconds": 300.0},
+        {"kind": "lost_contact", "label": "Lost contact", "started_at": now - timedelta(hours=3),
+         "ended_at": now - timedelta(hours=2), "ongoing": False, "seconds": 3600.0},
+    ]
+    async with app.test_request_context("/plugins/host_agent/panel/h1"):
+        session["user_role"] = "viewer"
+        page = await render_plugin_template(
+            app, "host_agent", "panel.html", host=host, reg=reg, reporting=True, stale=False,
+            target=None, ansible_available=False, managed_key_set=False, downtime=downtime)
+        quiet = await render_plugin_template(
+            app, "host_agent", "panel.html", host=host, reg=reg, reporting=True, stale=False,
+            target=None, ansible_available=False, managed_key_set=False, downtime=[])
+    assert "Downtime, last 7 days" in page
+    assert page.index("Rebooted") < page.index("Lost contact")
+    assert "going on, 5m" in page and "1h" in page
+    assert "Downtime, last 7 days" not in quiet

tests/integration/test_host_downtime.py

@@ -0,0 +1,146 @@
+"""Integration (#5756): what a host's agent says happened while it was away
+becomes the host's downtime, once, whatever the agent retries.
+
+The registration is seeded and the rows read with plain SQL: importing the
+plugin's models again after the app loaded the plugin raises "Table already
+defined" (see test_host_agent_alerts)."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _agent(app, last_seen: datetime):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+    tag = uuid.uuid4().hex[:8]
+    token = f"tok-{tag}-{uuid.uuid4().hex}"
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            host = Host(name=f"down-{tag}", address="10.9.9.8")
+            s.add(host)
+            await s.flush()
+            await s.execute(text(
+                "INSERT INTO host_agent_registrations "
+                "(id, host_id, token_hash, token_created_at, last_seen_at, created_at, updated_at) "
+                "VALUES (:id, :host, :hash, now(), :seen, now(), now())"),
+                {"id": str(uuid.uuid4()), "host": host.id, "seen": last_seen,
+                 "hash": hashlib.sha256(token.encode()).hexdigest()})
+            host_id = host.id
+    return host_id, token
+
+
+async def _post(app, token, samples, events):
+    resp = await app.test_client().post(
+        "/plugins/host_agent/ingest", json={"samples": samples, "events": events},
+        headers={"Authorization": f"Bearer {token}"})
+    assert resp.status_code == 200, await resp.get_data(as_text=True)
+
+
+async def _rows(app, host_id):
+    from sqlalchemy import text
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            return (await s.execute(text(
+                "SELECT kind, started_at, ended_at FROM host_downtime "
+                "WHERE host_id = :h ORDER BY started_at"), {"h": host_id})).all()
+
+
+def test_a_reboot_the_agent_announced_is_one_reboot_with_its_length(app):
+    async def _go():
+        now = datetime.now(timezone.utc)
+        stop, boot, back = now - timedelta(minutes=5), now - timedelta(minutes=3), now - timedelta(minutes=2)
+        host_id, token = await _agent(app, stop)
+        stopping = {"kind": "stopping", "at": stop.isoformat(), "host_shutdown": True}
+        # The stopping agent's last post carries no sample, and is retried.
+        await _post(app, token, [], [stopping])
+        await _post(app, token, [], [stopping])
+        assert [(k, e) for k, _, e in await _rows(app, host_id)] == [("reboot", None)]
+
+        started = {"kind": "started", "at": back.isoformat(), "boot_at": boot.isoformat()}
+        await _post(app, token, [{"ts": back.isoformat(), "cpu_pct": 3.0}], [started])
+        assert await _rows(app, host_id) == [("reboot", stop, back)]
+
+    asyncio.run(_go())
+
+
+def test_an_agent_restart_on_a_host_that_stayed_up_is_the_agent(app):
+    async def _go():
+        now = datetime.now(timezone.utc)
+        stop, back = now - timedelta(seconds=40), now - timedelta(seconds=30)
+        host_id, token = await _agent(app, stop)
+        # systemd couldn't say; the boot time, days before the stop, settles it.
+        await _post(app, token, [], [{"kind": "stopping", "at": stop.isoformat(), "host_shutdown": None}])
+        await _post(app, token, [{"ts": back.isoformat(), "cpu_pct": 3.0}],
+                    [{"kind": "started", "at": back.isoformat(),
+                      "boot_at": (now - timedelta(days=3)).isoformat()}])
+        assert await _rows(app, host_id) == [("agent_stop", stop, back)]
+
+    asyncio.run(_go())
+
+
+def test_a_boot_nobody_announced_is_a_restart_without_a_shutdown(app):
+    async def _go():
+        now = datetime.now(timezone.utc)
+        last, boot, back = now - timedelta(minutes=20), now - timedelta(minutes=4), now - timedelta(minutes=3)
+        host_id, token = await _agent(app, last)
+        started = {"kind": "started", "at": back.isoformat(), "boot_at": boot.isoformat()}
+        await _post(app, token, [{"ts": back.isoformat(), "cpu_pct": 3.0}], [started])
+        await _post(app, token, [{"ts": now.isoformat(), "cpu_pct": 3.0}], [started])
+        assert await _rows(app, host_id) == [("unclean", last, back)]
+
+    asyncio.run(_go())
+
+
+def test_lost_contact_is_the_span_the_agent_reports(app):
+    async def _go():
+        now = datetime.now(timezone.utc)
+        since, until = now - timedelta(minutes=45), now
+        host_id, token = await _agent(app, since)
+        lost = {"kind": "lost_contact", "from": since.isoformat(), "to": until.isoformat()}
+        await _post(app, token, [{"ts": until.isoformat(), "cpu_pct": 3.0}], [lost])
+        await _post(app, token, [{"ts": until.isoformat(), "cpu_pct": 3.0}], [lost])
+        assert await _rows(app, host_id) == [("lost_contact", since, until)]
+
+    asyncio.run(_go())
+
+
+def test_a_host_rebooting_inside_its_grace_is_silent_with_reason(app):
+    """Its other rules' silence is its agent's, as when it is offline (#5721),
+    until the grace runs out; then agent offline speaks for it."""
+    from steward.core.alerts import offline_agents
+    from steward.models.hosts import Host
+    from sqlalchemy import select
+
+    async def _go():
+        now = datetime.now(timezone.utc)
+        host_id, token = await _agent(app, now - timedelta(minutes=1))
+        await _post(app, token, [], [{"kind": "stopping", "at": (now - timedelta(minutes=1)).isoformat(),
+                                      "host_shutdown": True}])
+        old_id, old_token = await _agent(app, now - timedelta(hours=2))
+        await _post(app, old_token, [], [{"kind": "stopping", "at": (now - timedelta(hours=1)).isoformat(),
+                                          "host_shutdown": True}])
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                away = await offline_agents(s)
+                names = dict((await s.execute(select(Host.id, Host.name).where(
+                    Host.id.in_([host_id, old_id])))).all())
+        assert names[host_id] in away
+        assert names[old_id] not in away       # an hour on: past its 10-minute grace
+
+    asyncio.run(_go())

tests/plugins/host_agent/test_agent_events.py

@@ -0,0 +1,57 @@
+"""The agent says what happened to its host (#5756): when it started and the
+host booted, when it stops and whether the host is going down with it, and how
+long it couldn't reach Steward."""
+import subprocess
+from datetime import datetime
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from plugins.host_agent import agent as a
+
+
+def test_events_ride_the_payload_only_when_there_are_some():
+    assert "events" not in a.build_payload([], "h", {})
+    assert a.build_payload([], "h", {}, events=[{"kind": "started"}])["events"] == [{"kind": "started"}]
+
+
+def test_started_says_when_the_host_booted():
+    with patch.object(a, "_read_file", return_value="3600.5 100.0\n"), \
+         patch.object(a.time, "time", return_value=1_800_000_000.0):
+        event = a.started_event()
+    assert event["kind"] == "started"
+    assert datetime.fromisoformat(event["boot_at"]).timestamp() == 1_800_000_000.0 - 3600.5
+
+
+def test_started_without_uptime_still_says_it_started():
+    with patch.object(a, "_read_file", side_effect=OSError):
+        event = a.started_event()
+    assert event["kind"] == "started" and "boot_at" not in event
+
+
+def test_the_host_is_going_down_when_systemd_says_stopping():
+    def run(out):
+        return patch.object(a.subprocess, "run", return_value=SimpleNamespace(stdout=out))
+    with run("stopping\n"):
+        assert a.host_shutting_down() is True
+    with run("running\n"):
+        assert a.host_shutting_down() is False
+    with run("degraded\n"):
+        assert a.host_shutting_down() is False
+    with patch.object(a.subprocess, "run", side_effect=FileNotFoundError):
+        assert a.host_shutting_down() is None
+    with patch.object(a.subprocess, "run", side_effect=subprocess.TimeoutExpired("systemctl", 3)):
+        assert a.host_shutting_down() is None
+
+
+def test_stopping_says_whether_the_host_goes_with_it():
+    with patch.object(a, "host_shutting_down", return_value=True):
+        event = a.stopping_event()
+    assert event["kind"] == "stopping" and event["host_shutdown"] is True and event["at"]
+
+
+def test_lost_contact_is_no_answer_or_a_proxy_s_5xx_not_steward_refusing():
+    assert a.lost_contact(None) and a.lost_contact(502) and a.lost_contact(503)
+    assert not a.lost_contact(400) and not a.lost_contact(401)
+    event = a.lost_contact_event("2026-10-14T10:00:00+00:00")
+    assert event["kind"] == "lost_contact" and event["from"] == "2026-10-14T10:00:00+00:00"
+    assert event["to"] > event["from"]

tests/plugins/host_agent/test_downtime.py

@@ -0,0 +1,33 @@
+"""How a host's downtime reads on its page (#5756)."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+
+from plugins.host_agent.downtime import describe, restart_grace_seconds
+
+NOW = datetime(2026, 10, 14, 12, 0, tzinfo=timezone.utc)
+
+
+def _row(kind, minutes_ago, lasted=None):
+    start = NOW - timedelta(minutes=minutes_ago)
+    return SimpleNamespace(kind=kind, started_at=start,
+                           ended_at=start + timedelta(minutes=lasted) if lasted is not None else None)
+
+
+def test_each_kind_reads_as_what_it_was():
+    assert describe(_row("reboot", 30, 2), NOW)["label"] == "Rebooted"
+    assert describe(_row("unclean", 30, 9), NOW)["label"] == "Restarted without shutting down"
+    assert describe(_row("lost_contact", 30, 20), NOW)["label"] == "Lost contact"
+    assert describe(_row("agent_stop", 30, 1), NOW)["label"] == "Agent restarted"
+    assert describe(_row("agent_stop", 3), NOW)["label"] == "Agent stopped"
+
+
+def test_one_still_going_counts_to_now():
+    got = describe(_row("reboot", 4), NOW)
+    assert got["ongoing"] and got["seconds"] == 240.0
+    done = describe(_row("reboot", 30, 2), NOW)
+    assert not done["ongoing"] and done["seconds"] == 120.0
+
+
+def test_the_grace_comes_from_the_plugin_settings():
+    assert restart_grace_seconds({}) == 600
+    assert restart_grace_seconds({"PLUGINS": {"host_agent": {"reboot_grace_seconds": 120}}}) == 120

tests/plugins/host_agent/test_scheduler.py

@@ -91,3 +91,13 @@ def test_an_offline_agent_that_reports_after_the_restart_is_back():
     back = NOW - timedelta(minutes=1)
     regs = [FakeReg("h1", NOW - timedelta(seconds=15), NOW - timedelta(days=9))]
     assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [("nas", 15.0)]
+
+
+def test_a_reboot_the_agent_announced_reads_as_just_seen_until_its_grace_ends():
+    """#5756: an agent that said its host was rebooting isn't offline while
+    the reboot's grace lasts; another host silent as long still is."""
+    regs = [FakeReg("h1", NOW - timedelta(minutes=5), NOW - timedelta(days=9)),
+            FakeReg("h2", NOW - timedelta(minutes=5), NOW - timedelta(days=9))]
+    explained = {"h1": NOW + timedelta(minutes=5)}
+    assert seconds_since_seen(regs, NAMES, NOW, explained_until=explained) == [
+        ("nas", 0.0), ("web", 300.0)]