fix(gpu-jobs): end the error-tombstone loop — deliberate retry semantics + poison-job guards

The hourly ccip backfill's skip-list lacked 'error' (and the daily
siglip/embed variants re-gated failures on their missing results), so every
permanently-bad file got a fresh doomed job each run — ~24 duplicate error
rows/day per file, the perpetual 'unprocessable' flood. An errored job is now
a TOMBSTONE: no backfill re-enqueues it; retry is deliberate-only via
/retry_errors (an errored back-catalogue needs one button press after a
model swap).

One shared set of dedupe DELETEs (services/ml/gpu_jobs.error_dedupe_statements)
runs before every backfill and inside /retry_errors: error rows made moot by a
later pending/leased/done row go first, then older duplicates (newest reason
survives) — so the error count reads as distinct failing files and a retry
can't fan one file out into duplicate pending jobs. /retry_errors now returns
{requeued, pruned} and the toast shows both.

Poison-loop guards (release and lease-expiry burn no attempts, so a job that
stalls its transfer or crashes the agent every time cycled forever —
operator-observed jobs 99044/125288/131594/143131):
- agent: 3 in-session transient bounces (fetch or submit) → fail with the real
  reason instead of another release; strikes never count while stopping, and
  clear on submit success. Agent build 2026-07-02.3.
- server: the 60s orphan sweep (statements shared between the beat task and
  GpuJobService so they can't drift) converts expired leases with >=5 lease
  grants and pending jobs with >=10 to 'error', preserving the last stored
  failure reason. Backstops old agent builds.

Tests: tombstone rule across all three backfill variants, moot-row pruning,
poison conversions, and the extended /retry_errors dedupe contract.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM

tests/test_gpu_jobs.py

@@ -5,7 +5,11 @@ import pytest
 from sqlalchemy import func, select
 
 from backend.app.models import GpuJob, ImageRecord, ImageRegion
-from backend.app.services.ml.gpu_jobs import GpuJobService
+from backend.app.services.ml.gpu_jobs import (
+    EXPIRED_POISON_CAP,
+    PENDING_POISON_CAP,
+    GpuJobService,
+)
 
 pytestmark = pytest.mark.integration
 
@@ -263,3 +267,84 @@ async def test_recover_orphaned_resets_only_expired(db):
     await db.commit()
     assert (await db.get(GpuJob, expired.id)).status == "pending"
     assert (await db.get(GpuJob, fresh.id)).status == "leased"   # untouched
+
+
+@pytest.mark.asyncio
+async def test_backfill_skips_errored_images(db):
+    # An errored job is a TOMBSTONE for its (image, task): no backfill variant
+    # re-enqueues it — retry is deliberate-only (/retry_errors). Pre-fix, the
+    # hourly ccip run minted a fresh doomed job per bad file forever.
+    from backend.app.tasks.ml import enqueue_gpu_backfill
+
+    img = await _img(db, "f1" * 32)
+    svc = GpuJobService(db)
+    for task in ("ccip", "siglip", "embed"):
+        job = await svc.enqueue(img.id, task)
+        job.status = "error"
+        job.error = "no frames sampled from video"
+    await db.commit()
+
+    assert enqueue_gpu_backfill("ccip") == 0
+    assert enqueue_gpu_backfill("siglip") == 0
+    assert enqueue_gpu_backfill("embed") == 0
+
+
+@pytest.mark.asyncio
+async def test_backfill_prunes_moot_error_tombstones(db):
+    # Loop-era duplicates: several error rows for one (image, task), all made
+    # moot by a later done row. The backfill's dedupe pass removes them, and
+    # the done row still blocks re-enqueue.
+    from backend.app.tasks.ml import enqueue_gpu_backfill
+
+    img = await _img(db, "f2" * 32)
+    for i in range(3):
+        db.add(GpuJob(
+            image_record_id=img.id, task="ccip", status="error",
+            error=f"boom {i}",
+        ))
+    db.add(GpuJob(image_record_id=img.id, task="ccip", status="done"))
+    await db.commit()
+
+    assert enqueue_gpu_backfill("ccip") == 0
+
+    statuses = (await db.execute(
+        select(GpuJob.status).where(
+            GpuJob.image_record_id == img.id, GpuJob.task == "ccip"
+        )
+    )).scalars().all()
+    assert statuses == ["done"]
+
+
+@pytest.mark.asyncio
+async def test_recover_poisons_runaway_jobs(db):
+    # release/expiry loops never reach fail()'s attempt cap — the sweep converts
+    # them to 'error': an expired lease after EXPIRED_POISON_CAP grants (job
+    # crashes/wedges the agent every time) and a pending job after
+    # PENDING_POISON_CAP grants that never completed (transfer stalls forever).
+    img1 = await _img(db, "06" + "a" * 62)
+    img2 = await _img(db, "07" + "a" * 62)
+    svc = GpuJobService(db)
+    j1 = await svc.enqueue(img1.id, "ccip")
+    j2 = await svc.enqueue(img2.id, "ccip")
+    j1.status = "leased"
+    j1.lease_token = "dead"
+    j1.lease_expires_at = datetime.now(UTC) - timedelta(minutes=10)
+    j1.attempts = EXPIRED_POISON_CAP
+    j2.attempts = PENDING_POISON_CAP
+    j1_id = j1.id
+    j2_id = j2.id
+    await db.commit()
+
+    assert await svc.recover_orphaned() == 0   # nothing recovered — both poisoned
+    await db.commit()
+
+    # Column selects, not ORM refresh — the sweep wrote via Core DML.
+    rows = (await db.execute(
+        select(GpuJob.id, GpuJob.status, GpuJob.error)
+        .where(GpuJob.id.in_([j1_id, j2_id]))
+    )).all()
+    by_id = {r.id: r for r in rows}
+    assert by_id[j1_id].status == "error"
+    assert "poisoned" in by_id[j1_id].error
+    assert by_id[j2_id].status == "error"
+    assert "poisoned" in by_id[j2_id].error