feat(host_agent): a reboot reads as a reboot, and lost contact as its span (#5756)

#5756 "The agent says when its host rebooted and when it lost contact, so
downtime is recorded as what it was". Agent 1.14.0 posts events with the
next post that gets through: started (with the host's boot time), stopping
(on SIGTERM, with whether systemd says the host is going down) and
lost_contact (from its first unanswered post to the one carrying it).

Steward keeps them as the host's downtime (host_downtime): reboot,
agent_stop, unclean (a boot after the agent was last heard that nobody
announced) or lost_contact. Retried posts apply once.

An announced stop explains the silence for reboot_grace_seconds (Settings
-> Plugins -> host_agent, 10 minutes): seconds_since_seen reads 0 until
then, and the host's other rules go quiet with it (agents_away beside
#5721's registry). Past the grace, agent offline fires.

The host's Agent card lists the last 7 days of downtime. Rows are pruned
at data.retention_days.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -12,6 +12,7 @@ import re
 import shutil
 import signal
 import socket
+import subprocess
 import sys
 import time
 import urllib.error
@@ -20,7 +21,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.13.0"
+AGENT_VERSION = "1.14.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1330,16 +1331,70 @@ def build_sample(mounts: list[str], state: dict,
     return sample
 
 
-def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
+def build_payload(samples: list[dict], hostname: str, metadata: dict,
+                  events: list | None = None) -> dict:
     # sent_at is this host's clock at the moment of sending: Steward compares
     # it with its own on arrival to say how far apart the two clocks are.
-    return {
+    payload = {
         "agent_version": AGENT_VERSION,
         "hostname": hostname,
         "metadata": metadata,
         "sent_at": datetime.now(timezone.utc).isoformat(),
         "samples": samples,
     }
+    if events:
+        payload["events"] = events
+    return payload
+
+
+# ─── what happened to the host (#5756) ───────────────────────────────────────
+# Events ride the next post that gets through, so Steward records a reboot as a
+# reboot and a lost connection as the span it lasted, not as the agent gone.
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def started_event() -> dict:
+    """The agent starting, with when the host booted: Steward tells a host
+    that rebooted from an agent that only restarted by it."""
+    event = {"kind": "started", "at": _now_iso()}
+    try:
+        booted = time.time() - float(_read_file(UPTIME_PATH).split()[0])
+        event["boot_at"] = datetime.fromtimestamp(booted, timezone.utc).isoformat()
+    except Exception:
+        pass
+    return event
+
+
+def host_shutting_down():
+    """Whether the host itself is going down, not only this agent: systemd says
+    "stopping" while it shuts down or reboots. None when it can't tell."""
+    try:
+        out = subprocess.run(["systemctl", "is-system-running"], stdout=subprocess.PIPE,
+                             stderr=subprocess.DEVNULL, universal_newlines=True,
+                             timeout=3).stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return out == "stopping" if out else None
+
+
+def stopping_event() -> dict:
+    return {"kind": "stopping", "at": _now_iso(), "host_shutdown": host_shutting_down()}
+
+
+def lost_contact_event(since: str) -> dict:
+    """The span this agent could not reach Steward: from its first failed post
+    to now, the post that is carrying it."""
+    return {"kind": "lost_contact", "from": since, "to": _now_iso()}
+
+
+def lost_contact(status) -> bool:
+    """Whether a failed post means Steward couldn't be reached: no answer, or
+    a proxy's 5xx in front of a Steward that is down. A 400 or 401 is Steward
+    answering."""
+    return status is None or status >= 500
 
 
 # ─── POST + backoff ──────────────────────────────────────────────────────────
@@ -1441,6 +1496,9 @@ def main_loop(conf_path: str) -> int:
     docker_log_exclude = cfg.get("docker_log_exclude") or []
     buffer = RingBuffer(maxlen=20)
     backoff = 0
+    # Said once each, with the next post that gets through (#5756).
+    events: list = [started_event()]
+    lost_since = None
     # Carries previous net/disk counters + monotonic ts for rate computation.
     rate_state: dict = {}
 
@@ -1470,10 +1528,14 @@ def main_loop(conf_path: str) -> int:
             samples=send + [sample],
             hostname=hostname,
             metadata=metadata,
+            events=events + ([lost_contact_event(lost_since)] if lost_since else []),
         )
         ok, status = post_payload(cfg["url"], cfg["token"], payload)
 
         if ok:
+            if lost_since:
+                _log("INFO", f"reached Steward again (unreachable since {lost_since})")
+            events, lost_since = [], None
             for s in held:
                 buffer.push(s)
             if send:
@@ -1482,6 +1544,8 @@ def main_loop(conf_path: str) -> int:
             backoff = 0
             sleep_for = cfg["interval_seconds"]
         else:
+            if lost_since is None and lost_contact(status):
+                lost_since = sample["ts"]
             if status == 400:
                 _log("ERROR", "server rejected payload (400) — dropping sample")
             elif status == 401:
@@ -1501,10 +1565,12 @@ def main_loop(conf_path: str) -> int:
             slept += 1.0
 
     _log("INFO", "SIGTERM — flushing and exiting")
-    final = buffer.drain()
-    if final:
-        post_payload(cfg["url"], cfg["token"],
-                     build_payload(final, hostname, metadata))
+    # Say it is stopping, and whether the host is going down with it, so Steward
+    # reads the silence that follows as a reboot rather than the agent lost.
+    final_events = events + ([lost_contact_event(lost_since)] if lost_since else [])
+    final_events.append(stopping_event())
+    post_payload(cfg["url"], cfg["token"],
+                 build_payload(buffer.drain(), hostname, metadata, events=final_events))
     return 0