feat(triage): failed-processing triage — probe errored files, flag defects, recover (#125 C1-C3)

An errored GPU job's stored reason is a suspicion; the file probe is the
verdict. A 15-min beat sweep (triage_gpu_errors) runs verify_integrity's own
probe (sha256 + decode) on each errored image ONCE and writes both verdicts:
ImageRecord.integrity_status and the new GpuJob.triage_status ('defect' |
'file_ok', migration 0072). Every classification logs at WARNING so it
surfaces in Logs/System Activity.

- 'defect' rows are excluded from /retry_errors (re-running a known-bad file
  burns agent time re-minting the tombstone); response now reports
  defects_kept and the GpuAgentCard toast says so.
- GET /api/gpu/errors: triage view — reason buckets (classify_reason),
  probe verdicts, per-job detail. POST /errors/triage runs the sweep now.
- POST /api/gpu/errors/<id>/recover: reuses the Layer-2 refetch pattern —
  delete the defective copy + record (full cascade takes the tombstones too)
  and re-poll its subscription Source so a fresh copy re-imports and re-enters
  the pipeline; 'no_source' when nothing pollable resolves.
- New 'Failed processing' card (GpuTriageCard) in Maintenance: verdict counts,
  reason summary, probe-now, defect list with thumbnails + per-image Recover.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM

tests/test_api_gpu.py

@@ -213,3 +213,50 @@ async def test_retry_errors_requeues_only_errored(client, db):
 
     st = await (await client.get("/api/gpu/status")).get_json()
     assert st["pending"] == 1 and st["error"] == 0
+
+
+@pytest.mark.asyncio
+async def test_retry_errors_keeps_triaged_defects(client, db):
+    """A probe-confirmed DEFECT is a bad FILE — requeueing it just burns agent
+    time re-minting the tombstone, so /retry_errors leaves it for the recovery
+    surface and reports it as defects_kept."""
+    img1 = await _img(db, "4" * 64)
+    img2 = await _img(db, "5" * 64)
+    db.add(GpuJob(image_record_id=img1.id, task="ccip", status="error",
+                  attempts=3, error="moov atom not found",
+                  triage_status="defect"))
+    db.add(GpuJob(image_record_id=img2.id, task="ccip", status="error",
+                  attempts=3, error="ffmpeg timed out after 1200s"))
+    await db.commit()
+
+    body = await (await client.post("/api/gpu/retry_errors")).get_json()
+    assert body["requeued"] == 1
+    assert body["defects_kept"] == 1
+
+    rows = dict((await db.execute(
+        select(GpuJob.image_record_id, GpuJob.status)
+    )).all())
+    assert rows[img1.id] == "error"       # defect stays tombstoned
+    assert rows[img2.id] == "pending"     # operational failure requeued
+
+
+@pytest.mark.asyncio
+async def test_errors_endpoint_reports_triage_view(client, db):
+    img = await _img(db, "6" * 64)
+    db.add(GpuJob(image_record_id=img.id, task="ccip", status="error",
+                  attempts=3,
+                  error="no frames sampled from video — moov atom not found"))
+    await db.commit()
+
+    resp = await client.get("/api/gpu/errors")
+    assert resp.status_code == 200
+    body = await resp.get_json()
+    assert body["total"] == 1
+    assert body["by_class"] == {"truncated_or_corrupt": 1}
+    assert body["triage"]["unclassified"] == 1
+    item = body["items"][0]
+    assert item["image_id"] == img.id
+    assert item["task"] == "ccip"
+    assert item["reason_class"] == "truncated_or_corrupt"
+    assert item["triage_status"] is None
+    assert item["image_url"].startswith("/images/")

tests/test_gpu_triage.py

@@ -0,0 +1,165 @@
+"""Failure triage (#125): probe errored jobs' files, flag verdicts, recover.
+
+The probe is the arbiter: reason strings only bucket the overview. A file that
+passes checksum+decode is 'file_ok' (operational failure); anything else is a
+'defect' — surfaced for recovery and excluded from /retry_errors.
+"""
+import hashlib
+
+import pytest
+from PIL import Image as PILImage
+from sqlalchemy import select
+
+from backend.app.models import (
+    Artist,
+    GpuJob,
+    ImageProvenance,
+    ImageRecord,
+    Post,
+    Source,
+)
+from backend.app.services.ml.gpu_triage import (
+    classify_reason,
+    recover_defective_image,
+    triage_errored_jobs,
+)
+
+pytestmark = pytest.mark.integration
+
+
+def test_classify_reason_buckets():
+    assert classify_reason(
+        "no frames sampled from video — moov atom not found"
+    ) == "truncated_or_corrupt"
+    assert classify_reason("ffmpeg timed out after 1200s") == "timeout"
+    assert classify_reason(
+        "gave up after repeated transient failures: HTTPConnectionPool read timed out"
+    ) == "transient"
+    assert classify_reason(
+        "poisoned: 10+ lease attempts without ever completing"
+    ) == "poisoned"
+    assert classify_reason("cannot identify image file") == "decode"
+    assert classify_reason("something novel") == "other"
+    assert classify_reason(None) == "other"
+
+
+async def _errored_image(db, tmp_path, *, name, sha, content: bytes | None,
+                         error="no frames sampled from video — moov atom not found"):
+    """An ImageRecord (file written iff content is not None) + an errored job."""
+    path = tmp_path / name
+    if content is not None:
+        path.write_bytes(content)
+    img = ImageRecord(
+        path=str(path), sha256=sha, size_bytes=1, mime="image/png",
+        width=1, height=1, origin="imported_filesystem",
+        integrity_status="unknown",
+    )
+    db.add(img)
+    await db.flush()
+    db.add(GpuJob(image_record_id=img.id, task="ccip", status="error",
+                  error=error, attempts=3))
+    await db.flush()
+    return img
+
+
+@pytest.mark.asyncio
+async def test_triage_probes_and_splits_defect_vs_file_ok(db, tmp_path):
+    # Healthy: a real PNG whose recorded sha matches its bytes → file_ok.
+    ok_path = tmp_path / "fine.png"
+    PILImage.new("RGB", (4, 4), (200, 30, 30)).save(ok_path)
+    ok_sha = hashlib.sha256(ok_path.read_bytes()).hexdigest()
+    ok = await _errored_image(db, tmp_path, name="fine.png", sha=ok_sha,
+                              content=None, error="ffmpeg timed out after 1200s")
+    # Corrupt: bytes don't match the recorded sha → defect.
+    bad = await _errored_image(db, tmp_path, name="bad.png", sha="0" * 64,
+                               content=b"not a real png")
+    # Missing: no file on disk at all → defect (failed_verification).
+    gone = await _errored_image(db, tmp_path, name="gone.png", sha="1" * 64,
+                                content=None)
+    await db.commit()
+
+    summary = await db.run_sync(lambda s: triage_errored_jobs(s))
+    assert summary["probed"] == 3
+    assert summary["defect"] == 2
+    assert summary["file_ok"] == 1
+    assert summary["partial"] is False
+
+    # Column selects, not ORM refresh — the sweep wrote via Core DML.
+    rows = dict((await db.execute(
+        select(GpuJob.image_record_id, GpuJob.triage_status)
+        .where(GpuJob.status == "error")
+    )).all())
+    assert rows[ok.id] == "file_ok"
+    assert rows[bad.id] == "defect"
+    assert rows[gone.id] == "defect"
+    verdicts = dict((await db.execute(
+        select(ImageRecord.id, ImageRecord.integrity_status)
+        .where(ImageRecord.id.in_([ok.id, bad.id, gone.id]))
+    )).all())
+    assert verdicts[ok.id] == "ok"
+    assert verdicts[bad.id] == "corrupt"
+    assert verdicts[gone.id] == "failed_verification"
+
+    # Idempotent: everything already triaged → no re-probe.
+    again = await db.run_sync(lambda s: triage_errored_jobs(s))
+    assert again["probed"] == 0
+
+
+@pytest.mark.asyncio
+async def test_recover_without_pollable_source_reports_no_source(db, tmp_path):
+    img = await _errored_image(db, tmp_path, name="orphan.png", sha="2" * 64,
+                               content=b"x")
+    await db.commit()
+    res = await db.run_sync(
+        lambda s: recover_defective_image(s, img.id, images_root=tmp_path)
+    )
+    assert res["status"] == "no_source"
+    still_there = (await db.execute(
+        select(ImageRecord.id).where(ImageRecord.id == img.id)
+    )).scalar_one_or_none()
+    assert still_there == img.id
+
+
+@pytest.mark.asyncio
+async def test_recover_deletes_record_and_requeues_source(
+    client, db, tmp_path, monkeypatch,
+):
+    img = await _errored_image(db, tmp_path, name="fixme.png", sha="3" * 64,
+                               content=b"x")
+    artist = Artist(name="Recov", slug="recov")
+    db.add(artist)
+    await db.flush()
+    src = Source(artist_id=artist.id, platform="patreon",
+                 url="https://www.patreon.com/recov", enabled=True)
+    db.add(src)
+    await db.flush()
+    post = Post(artist_id=artist.id, source_id=src.id, external_post_id="p1")
+    db.add(post)
+    await db.flush()
+    db.add(ImageProvenance(image_record_id=img.id, post_id=post.id,
+                           source_id=src.id))
+    img_id, src_id = img.id, src.id
+    await db.commit()
+
+    queued = []
+    monkeypatch.setattr(
+        "backend.app.tasks.download.download_source.delay",
+        lambda sid: queued.append(sid),
+    )
+
+    resp = await client.post(f"/api/gpu/errors/{img_id}/recover")
+    assert resp.status_code == 200
+    body = await resp.get_json()
+    assert body["status"] == "refetch_queued"
+    assert body["source_id"] == src_id
+    assert queued == [src_id]
+
+    # Record gone — the error tombstones cascade away with it.
+    remaining = (await db.execute(
+        select(ImageRecord.id).where(ImageRecord.id == img_id)
+    )).scalar_one_or_none()
+    assert remaining is None
+    jobs_left = (await db.execute(
+        select(GpuJob.id).where(GpuJob.image_record_id == img_id)
+    )).scalars().all()
+    assert jobs_left == []