feat(gpu): fast orphan recovery — graceful release + 60s sweep (#114)

So work an agent orphaned gets picked back up quickly, three layers:
- GpuJobService.release(): a graceful agent stop hands its still-leased jobs back
  to pending instantly (POST /api/gpu/jobs/release), no waiting out the lease.
- GpuJobService.recover_orphaned() + recover_orphaned_gpu_jobs Celery task on a
  60s beat: resets expired leases (a hard-crashed agent) to pending and keeps the
  queue counts honest even when nothing is leasing.
- Lease TTL 300→180s: still well above any single job (a capped-frame video embed
  is tens of seconds, and a live worker heartbeats), but a hard crash recovers
  faster once the sweep fires.

Tests: release returns-to-pending (token-scoped), recover_orphaned resets only
expired leases, release API round-trip.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

tests/test_api_gpu.py

@@ -96,3 +96,22 @@ async def test_backfill_enqueues_then_is_idempotent(db):
     n = enqueue_gpu_backfill("ccip")     # sync task, own session
     assert n >= 2
     assert enqueue_gpu_backfill("ccip") == 0   # all already pending
+
+
+@pytest.mark.asyncio
+async def test_release_hands_job_back_to_pending(client, db):
+    img = await _img(db, "e" * 64)
+    await GpuJobService(db).enqueue(img.id, "ccip")
+    await db.commit()
+    token = (await (await client.post("/api/gpu/token/rotate")).get_json())["token"]
+    hdr = {"Authorization": f"Bearer {token}"}
+    j = (await (await client.post(
+        "/api/gpu/jobs/lease", json={"agent_id": "a1"}, headers=hdr,
+    )).get_json())["jobs"][0]
+
+    resp = await client.post("/api/gpu/jobs/release", json={
+        "agent_id": "a1", "job_ids": [j["job_id"]],
+    }, headers=hdr)
+    assert resp.status_code == 200 and (await resp.get_json())["released"] == 1
+    st = await (await client.get("/api/gpu/status")).get_json()
+    assert st["pending"] == 1 and st["leased"] == 0

tests/test_gpu_jobs.py

@@ -123,3 +123,38 @@ async def test_fail_requeues_until_cap(db):
     assert await svc.fail("agent-1", job.id, "boom again") is True
     await db.commit()
     assert (await db.get(GpuJob, job.id)).status == "error"
+
+
+@pytest.mark.asyncio
+async def test_release_returns_to_pending(db):
+    img = await _img(db, "01" + "a" * 62)
+    svc = GpuJobService(db)
+    await svc.enqueue(img.id, "ccip")
+    await db.commit()
+    job = (await svc.lease("agent-1"))[0]
+    await db.commit()
+
+    assert await svc.release("other", [job.id]) == 0      # not this token's lease
+    assert await svc.release("agent-1", [job.id]) == 1     # graceful hand-back
+    await db.commit()
+    fresh = await db.get(GpuJob, job.id)
+    assert fresh.status == "pending" and fresh.lease_token is None
+
+
+@pytest.mark.asyncio
+async def test_recover_orphaned_resets_only_expired(db):
+    img1 = await _img(db, "02" + "a" * 62)
+    img2 = await _img(db, "03" + "a" * 62)
+    svc = GpuJobService(db)
+    await svc.enqueue(img1.id, "ccip")
+    await svc.enqueue(img2.id, "ccip")
+    await db.commit()
+    expired, fresh = await svc.lease("dead", batch_size=2)
+    # One lease is in the past (orphaned), the other still valid.
+    expired.lease_expires_at = datetime.now(UTC) - timedelta(minutes=10)
+    await db.commit()
+
+    assert await svc.recover_orphaned() == 1
+    await db.commit()
+    assert (await db.get(GpuJob, expired.id)).status == "pending"
+    assert (await db.get(GpuJob, fresh.id)).status == "leased"   # untouched