Release: dev → main (first public release) #258

Merged
bvandeusen merged 94 commits from dev into main 2026-09-25 10:02:40 -04:00
2 changed files with 98 additions and 5 deletions
Showing only changes of commit 693759f2bb - Show all commits
+33 -4
View File
@@ -342,14 +342,43 @@ class Worker:
# --- background loops ---------------------------------------------------
def _heartbeat_loop(self) -> None:
"""Keep every held lease alive so buffered jobs waiting on the GPU aren't
reclaimed by curator's 180s TTL. Errors are swallowed by client.heartbeat;
a reclaimed lease just re-leases elsewhere — never fatal."""
"""Keep every held lease alive, and say we are here even when holding none.
Leases: buffered jobs waiting on the GPU would otherwise be reclaimed by
curator's 180s TTL. Errors are swallowed by client.heartbeat; a reclaimed
lease just re-leases elsewhere — never fatal.
## Why this sends with an EMPTY list rather than skipping
Curator's roster records a check-in on this call (and on `lease`), and
calls an agent stopped after 300s of silence. This loop used to be
gated on `if ids:` — so an agent holding no leases sent nothing at all,
and the only check-in left was the lease poll, which sleep mode backs
off exponentially to a 900s ceiling (see IDLE_POLL_MAX_SECONDS).
900 against 300: an IDLE agent was structurally guaranteed to read as
stopped. Operator, 2026-09-23: *"I'm running the gpu agent on my device
and it currently reads as 'offline' but it's running and has checked in
recently."* It had — twelve minutes ago, partway up the backoff ladder.
The two halves were written ten weeks apart and never reconciled: sleep
mode landed 2026-07-02, and the roster adopted the lease as its
check-in on 2026-09-02 without noticing the call it was piggybacking on
had been deliberately slowed.
An empty heartbeat extends nothing (`id.in_([])` matches no rows) and
costs one small POST every 45s — against the 6/min lease poll sleep
mode exists to avoid, that is not a cadence worth protecting, and it is
what makes "is the agent alive" answerable at all.
Still gated on `self._running`: a worker that has been stopped is not
checking in for work, and reporting it as present would be a different
lie.
"""
while True:
if self._running:
with self._held_lock:
ids = list(self._held)
if ids:
self.client.heartbeat(ids)
time.sleep(HEARTBEAT_INTERVAL)
+64
View File
@@ -314,3 +314,67 @@ async def test_cpu_embed_never_blocks_gpu_crop_backfills(db):
select(GpuJob.task).where(GpuJob.image_record_id == img.id)
)).scalars().all())
assert tasks == {"ccip", "siglip"}
# --- an idle agent still checks in -------------------------------------------
@pytest.mark.asyncio
async def test_a_heartbeat_with_no_jobs_still_records_the_check_in(client, db):
"""Operator, 2026-09-23: *"I'm running the gpu agent on my device and it
currently reads as 'offline' but it's running and has checked in
recently."*
It had — twelve minutes ago. The roster takes its agent check-in from the
`lease` and `heartbeat` calls, and calls an agent stopped after 300s of
silence. The agent's heartbeat loop was gated on holding leases, so an
IDLE agent sent none; the only check-in left was the lease poll, which
sleep mode backs off to a 900s ceiling. 900 against 300 — an idle agent
was structurally guaranteed to read as stopped.
So the empty heartbeat has to be a real check-in on the server side, not
merely tolerated. Asserted on `last_seen_at` moving, because "it returned
200" would pass against an endpoint that recorded nothing.
"""
from backend.app.models import ServiceSeen
token = (await (await client.post("/api/gpu/token/rotate")).get_json())["token"]
hdr = {"Authorization": f"Bearer {token}"}
resp = await client.post(
"/api/gpu/jobs/heartbeat",
json={"agent_id": "desktop-agent", "job_ids": []}, headers=hdr,
)
assert resp.status_code == 200
assert (await resp.get_json())["extended"] == 0, "it extends no lease"
row = (await db.execute(
select(ServiceSeen).where(ServiceSeen.key == "agent:desktop-agent")
)).scalar_one()
assert row.kind == "agent"
assert row.display_name == "GPU agent (desktop-agent)"
assert row.last_seen_at is not None
def test_the_agent_heartbeats_whether_or_not_it_holds_a_lease():
"""The other half, in the agent itself — the half that was actually wrong.
Read from the source rather than by running the loop: it is a `while True`
with a sleep, so exercising it means threads and timing, and the property
is simply that the call is not behind a `if ids:`.
"""
from pathlib import Path
src = (
Path(__file__).resolve().parents[1] / "agent" / "fc_agent" / "worker.py"
).read_text()
loop = src[src.index("def _heartbeat_loop"):]
loop = loop[:loop.index("\n def ")]
assert "self.client.heartbeat(ids)" in loop
assert "if ids:" not in loop, (
"the heartbeat is gated on holding leases again; an idle agent then "
"reads as stopped after 300s while sleep mode backs its lease poll "
"off to 900s"
)