feat(triage): failed-processing triage — probe errored files, flag defects, recover (#125 C1-C3)
An errored GPU job's stored reason is a suspicion; the file probe is the
verdict. A 15-min beat sweep (triage_gpu_errors) runs verify_integrity's own
probe (sha256 + decode) on each errored image ONCE and writes both verdicts:
ImageRecord.integrity_status and the new GpuJob.triage_status ('defect' |
'file_ok', migration 0072). Every classification logs at WARNING so it
surfaces in Logs/System Activity.
- 'defect' rows are excluded from /retry_errors (re-running a known-bad file
burns agent time re-minting the tombstone); response now reports
defects_kept and the GpuAgentCard toast says so.
- GET /api/gpu/errors: triage view — reason buckets (classify_reason),
probe verdicts, per-job detail. POST /errors/triage runs the sweep now.
- POST /api/gpu/errors/<id>/recover: reuses the Layer-2 refetch pattern —
delete the defective copy + record (full cascade takes the tombstones too)
and re-poll its subscription Source so a fresh copy re-imports and re-enters
the pipeline; 'no_source' when nothing pollable resolves.
- New 'Failed processing' card (GpuTriageCard) in Maintenance: verdict counts,
reason summary, probe-now, defect list with thumbnails + per-image Recover.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -213,3 +213,50 @@ async def test_retry_errors_requeues_only_errored(client, db):
|
||||
|
||||
st = await (await client.get("/api/gpu/status")).get_json()
|
||||
assert st["pending"] == 1 and st["error"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_errors_keeps_triaged_defects(client, db):
|
||||
"""A probe-confirmed DEFECT is a bad FILE — requeueing it just burns agent
|
||||
time re-minting the tombstone, so /retry_errors leaves it for the recovery
|
||||
surface and reports it as defects_kept."""
|
||||
img1 = await _img(db, "4" * 64)
|
||||
img2 = await _img(db, "5" * 64)
|
||||
db.add(GpuJob(image_record_id=img1.id, task="ccip", status="error",
|
||||
attempts=3, error="moov atom not found",
|
||||
triage_status="defect"))
|
||||
db.add(GpuJob(image_record_id=img2.id, task="ccip", status="error",
|
||||
attempts=3, error="ffmpeg timed out after 1200s"))
|
||||
await db.commit()
|
||||
|
||||
body = await (await client.post("/api/gpu/retry_errors")).get_json()
|
||||
assert body["requeued"] == 1
|
||||
assert body["defects_kept"] == 1
|
||||
|
||||
rows = dict((await db.execute(
|
||||
select(GpuJob.image_record_id, GpuJob.status)
|
||||
)).all())
|
||||
assert rows[img1.id] == "error" # defect stays tombstoned
|
||||
assert rows[img2.id] == "pending" # operational failure requeued
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_errors_endpoint_reports_triage_view(client, db):
|
||||
img = await _img(db, "6" * 64)
|
||||
db.add(GpuJob(image_record_id=img.id, task="ccip", status="error",
|
||||
attempts=3,
|
||||
error="no frames sampled from video — moov atom not found"))
|
||||
await db.commit()
|
||||
|
||||
resp = await client.get("/api/gpu/errors")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["total"] == 1
|
||||
assert body["by_class"] == {"truncated_or_corrupt": 1}
|
||||
assert body["triage"]["unclassified"] == 1
|
||||
item = body["items"][0]
|
||||
assert item["image_id"] == img.id
|
||||
assert item["task"] == "ccip"
|
||||
assert item["reason_class"] == "truncated_or_corrupt"
|
||||
assert item["triage_status"] is None
|
||||
assert item["image_url"].startswith("/images/")
|
||||
|
||||
Reference in New Issue
Block a user