fix(gpu-jobs): end the error-tombstone loop — deliberate retry semantics + poison-job guards

The hourly ccip backfill's skip-list lacked 'error' (and the daily
siglip/embed variants re-gated failures on their missing results), so every
permanently-bad file got a fresh doomed job each run — ~24 duplicate error
rows/day per file, the perpetual 'unprocessable' flood. An errored job is now
a TOMBSTONE: no backfill re-enqueues it; retry is deliberate-only via
/retry_errors (an errored back-catalogue needs one button press after a
model swap).

One shared set of dedupe DELETEs (services/ml/gpu_jobs.error_dedupe_statements)
runs before every backfill and inside /retry_errors: error rows made moot by a
later pending/leased/done row go first, then older duplicates (newest reason
survives) — so the error count reads as distinct failing files and a retry
can't fan one file out into duplicate pending jobs. /retry_errors now returns
{requeued, pruned} and the toast shows both.

Poison-loop guards (release and lease-expiry burn no attempts, so a job that
stalls its transfer or crashes the agent every time cycled forever —
operator-observed jobs 99044/125288/131594/143131):
- agent: 3 in-session transient bounces (fetch or submit) → fail with the real
  reason instead of another release; strikes never count while stopping, and
  clear on submit success. Agent build 2026-07-02.3.
- server: the 60s orphan sweep (statements shared between the beat task and
  GpuJobService so they can't drift) converts expired leases with >=5 lease
  grants and pending jobs with >=10 to 'error', preserving the last stored
  failure reason. Backstops old agent builds.

Tests: tombstone rule across all three backfill variants, moot-row pruning,
poison conversions, and the extended /retry_errors dedupe contract.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM

tests/test_api_gpu.py

@@ -1,6 +1,8 @@
 """GPU-job HTTP API (#114): bearer auth + lease/submit round-trip + backfill."""
+from datetime import UTC, datetime, timedelta
+
 import pytest
-from sqlalchemy import select
+from sqlalchemy import func, select
 
 from backend.app.models import GpuJob, ImageRecord
 from backend.app.services.ml.gpu_jobs import GpuJobService
@@ -153,8 +155,11 @@ async def test_release_hands_job_back_to_pending(client, db):
 
 @pytest.mark.asyncio
 async def test_retry_errors_requeues_only_errored(client, db):
-    """/retry_errors resets ERRORED jobs (any task) to pending with a fresh
-    retry budget — and leaves done work untouched (it is NOT /reprocess)."""
+    """/retry_errors prunes stale tombstones first (older duplicates + rows a
+    later success made moot), then resets the SURVIVING errored jobs to pending
+    with a fresh retry budget — and leaves done work untouched (NOT /reprocess).
+    The prune is what stops one failing file fanning out into duplicate pending
+    jobs (the 2026-07-02 tombstone loop minted one error row per hour)."""
     img1 = await _img(db, "1" * 64)
     img2 = await _img(db, "2" * 64)
     svc = GpuJobService(db)
@@ -165,12 +170,30 @@ async def test_retry_errors_requeues_only_errored(client, db):
     j_err.status = "error"
     j_err.attempts = 3
     j_err.error = "no frames sampled from video (unprocessable)"
+    j_err.updated_at = datetime.now(UTC)
     j_done.status = "done"
+    # Loop-era leftovers: an OLDER duplicate error row for img1's ccip, and a
+    # tombstone img2's done row makes moot — both pruned, never requeued.
+    dup = GpuJob(
+        image_record_id=img1.id, task="ccip", status="error",
+        error="older duplicate", updated_at=datetime.now(UTC) - timedelta(hours=1),
+    )
+    moot = GpuJob(
+        image_record_id=img2.id, task="siglip", status="error",
+        error="superseded by the done row",
+    )
+    db.add(dup)
+    db.add(moot)
+    await db.flush()
+    dup_id = dup.id
+    moot_id = moot.id
     await db.commit()
 
     resp = await client.post("/api/gpu/retry_errors")
     assert resp.status_code == 200
-    assert (await resp.get_json())["requeued"] == 1
+    body = await resp.get_json()
+    assert body["requeued"] == 1
+    assert body["pruned"] == 2
 
     # Column selects, not ORM refresh — the route wrote via Core DML.
     row = (await db.execute(
@@ -182,6 +205,11 @@ async def test_retry_errors_requeues_only_errored(client, db):
         select(GpuJob.status).where(GpuJob.id == done_id)
     )
     assert done_status == "done"
+    survivors = (await db.execute(
+        select(func.count()).select_from(GpuJob)
+        .where(GpuJob.id.in_([dup_id, moot_id]))
+    )).scalar_one()
+    assert survivors == 0
 
     st = await (await client.get("/api/gpu/status")).get_json()
     assert st["pending"] == 1 and st["error"] == 0

tests/test_gpu_jobs.py

@@ -5,7 +5,11 @@ import pytest
 from sqlalchemy import func, select
 
 from backend.app.models import GpuJob, ImageRecord, ImageRegion
-from backend.app.services.ml.gpu_jobs import GpuJobService
+from backend.app.services.ml.gpu_jobs import (
+    EXPIRED_POISON_CAP,
+    PENDING_POISON_CAP,
+    GpuJobService,
+)
 
 pytestmark = pytest.mark.integration
 
@@ -263,3 +267,84 @@ async def test_recover_orphaned_resets_only_expired(db):
     await db.commit()
     assert (await db.get(GpuJob, expired.id)).status == "pending"
     assert (await db.get(GpuJob, fresh.id)).status == "leased"   # untouched
+
+
+@pytest.mark.asyncio
+async def test_backfill_skips_errored_images(db):
+    # An errored job is a TOMBSTONE for its (image, task): no backfill variant
+    # re-enqueues it — retry is deliberate-only (/retry_errors). Pre-fix, the
+    # hourly ccip run minted a fresh doomed job per bad file forever.
+    from backend.app.tasks.ml import enqueue_gpu_backfill
+
+    img = await _img(db, "f1" * 32)
+    svc = GpuJobService(db)
+    for task in ("ccip", "siglip", "embed"):
+        job = await svc.enqueue(img.id, task)
+        job.status = "error"
+        job.error = "no frames sampled from video"
+    await db.commit()
+
+    assert enqueue_gpu_backfill("ccip") == 0
+    assert enqueue_gpu_backfill("siglip") == 0
+    assert enqueue_gpu_backfill("embed") == 0
+
+
+@pytest.mark.asyncio
+async def test_backfill_prunes_moot_error_tombstones(db):
+    # Loop-era duplicates: several error rows for one (image, task), all made
+    # moot by a later done row. The backfill's dedupe pass removes them, and
+    # the done row still blocks re-enqueue.
+    from backend.app.tasks.ml import enqueue_gpu_backfill
+
+    img = await _img(db, "f2" * 32)
+    for i in range(3):
+        db.add(GpuJob(
+            image_record_id=img.id, task="ccip", status="error",
+            error=f"boom {i}",
+        ))
+    db.add(GpuJob(image_record_id=img.id, task="ccip", status="done"))
+    await db.commit()
+
+    assert enqueue_gpu_backfill("ccip") == 0
+
+    statuses = (await db.execute(
+        select(GpuJob.status).where(
+            GpuJob.image_record_id == img.id, GpuJob.task == "ccip"
+        )
+    )).scalars().all()
+    assert statuses == ["done"]
+
+
+@pytest.mark.asyncio
+async def test_recover_poisons_runaway_jobs(db):
+    # release/expiry loops never reach fail()'s attempt cap — the sweep converts
+    # them to 'error': an expired lease after EXPIRED_POISON_CAP grants (job
+    # crashes/wedges the agent every time) and a pending job after
+    # PENDING_POISON_CAP grants that never completed (transfer stalls forever).
+    img1 = await _img(db, "06" + "a" * 62)
+    img2 = await _img(db, "07" + "a" * 62)
+    svc = GpuJobService(db)
+    j1 = await svc.enqueue(img1.id, "ccip")
+    j2 = await svc.enqueue(img2.id, "ccip")
+    j1.status = "leased"
+    j1.lease_token = "dead"
+    j1.lease_expires_at = datetime.now(UTC) - timedelta(minutes=10)
+    j1.attempts = EXPIRED_POISON_CAP
+    j2.attempts = PENDING_POISON_CAP
+    j1_id = j1.id
+    j2_id = j2.id
+    await db.commit()
+
+    assert await svc.recover_orphaned() == 0   # nothing recovered — both poisoned
+    await db.commit()
+
+    # Column selects, not ORM refresh — the sweep wrote via Core DML.
+    rows = (await db.execute(
+        select(GpuJob.id, GpuJob.status, GpuJob.error)
+        .where(GpuJob.id.in_([j1_id, j2_id]))
+    )).all()
+    by_id = {r.id: r for r in rows}
+    assert by_id[j1_id].status == "error"
+    assert "poisoned" in by_id[j1_id].error
+    assert by_id[j2_id].status == "error"
+    assert "poisoned" in by_id[j2_id].error