feat(host_agent): agent 1.10 — backoff inside the stale window, posts of five, a deadline on Docker, /system/df every ten minutes (#5154)

- BACKOFF_CAP 300 → 60: Steward reads a host stale after 180 s, so an agent
  backing off to five minutes read stopped long after Steward came back
  (lesson #4355: a check-in inherits its call's backoff).
- A POST carries at most five samples: the oldest four buffered plus the
  new one, the rest held for the next post. A backlog drains a few at a
  time instead of in one post Steward cannot write before the timeout
  (the loop measured in #5149).
- Per-container Docker work has a 20 s deadline over the whole fan-out.
  Past it the sample reports docker_status "error" ("N of M containers
  not read within 20 s") and no list: a partial list would read as
  removals, and a record without its inspect as restarts.
- /system/df is asked every 600 s rather than every sample; Steward keeps
  the last answer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/host_agent/test_agent_build_payload.py

@@ -113,11 +113,18 @@ def test_post_payload_network_error_returns_none_status():
     assert status is None
 
 
-def test_backoff_schedule():
+def test_backoff_schedule_stays_inside_the_stale_window():
     from plugins.host_agent.agent import next_backoff
     assert next_backoff(0) == 30
     assert next_backoff(30) == 60
-    assert next_backoff(60) == 120
-    assert next_backoff(120) == 240
-    assert next_backoff(240) == 300  # capped
-    assert next_backoff(300) == 300
+    assert next_backoff(60) == 60    # capped below Steward's 180 s stale window (#5154)
+
+
+def test_a_backlog_drains_a_few_samples_a_post_oldest_first():
+    from plugins.host_agent.agent import MAX_SAMPLES_PER_POST, split_backlog
+    buffered = list(range(20))
+    send, held = split_backlog(buffered)
+    assert send == [0, 1, 2, 3] and held == list(range(4, 20))
+    assert len(send) + 1 == MAX_SAMPLES_PER_POST    # room for the new sample
+    assert split_backlog([7]) == ([7], [])
+    assert split_backlog([]) == ([], [])

tests/plugins/host_agent/test_agent_docker.py

@@ -706,3 +706,39 @@ def test_read_config_parses_docker_log_exclude(tmp_path):
 ])
 def test_docker_started_at(insp, started):
     assert a._docker_started_at(insp) == started
+
+
+def test_collect_docker_past_its_deadline_says_so_and_lists_nothing(monkeypatch):
+    # A partial list would read as removals; a record without inspect as restarts.
+    import threading
+    release = threading.Event()
+
+    def one(socket_path, c):
+        if c["Names"] == ["/stuck"]:
+            release.wait(2)
+        return {"name": c["Names"][0].lstrip("/")}
+
+    listing = [{"Id": "a" * 16, "Names": ["/fine"]}, {"Id": "b" * 16, "Names": ["/stuck"]}]
+    monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: listing)
+    monkeypatch.setattr(a, "_collect_one_container", one)
+    monkeypatch.setattr(a, "DOCKER_DEADLINE_SECONDS", 0.05)
+    try:
+        assert a.collect_docker("/s") == ("error", "1 of 2 containers not read within 0.05 s", [])
+    finally:
+        release.set()
+
+
+def test_disk_usage_is_asked_on_its_own_cadence(monkeypatch):
+    calls = []
+    clock = {"t": 1000.0}
+    monkeypatch.setattr(a.time, "monotonic", lambda: clock["t"])
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: calls.append(clock["t"]) or {"images_total": 1})
+    state = {}
+    seen = []
+    for t in (1000.0, 1030.0, 1000.0 + a.DOCKER_DF_INTERVAL_SECONDS):
+        clock["t"] = t
+        seen.append("docker_disk" in a.build_sample(["/"], state, "/s"))
+    assert seen == [True, False, True]
+    assert calls == [1000.0, 1000.0 + a.DOCKER_DF_INTERVAL_SECONDS]