feat(gpu): "Retry errored jobs" — scoped requeue of errors only
After an agent-side fix (e.g. the short-video sampler), the errored jobs
(~2.8k) have exhausted their 3 attempts and stay parked: backfill skips
images that already have a job, and /reprocess is the nuclear option (it
resets the 179k DONE jobs too). There was no way to re-run just the errors.
POST /api/gpu/retry_errors resets every status='error' job (all task types)
to pending with attempts=0 and the stored error cleared — a small inline
UPDATE that returns {requeued: n} so the UI toast can show the count.
UI: a "Retry errored jobs" button on the GPU-agent card, right under the
queue tiles; disabled when errored==0. With the agent now logging ffmpeg's
stderr on failure, retrying also reveals which errors were real vs victims
of the fps-filter bug.
Test: retry_errors requeues the errored job (fresh attempts, error cleared)
and leaves done work untouched; asserts via column selects (Core-DML
gotcha), not ORM refresh.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+38
-1
@@ -1,7 +1,8 @@
|
||||
"""GPU-job HTTP API (#114): bearer auth + lease/submit round-trip + backfill."""
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import ImageRecord
|
||||
from backend.app.models import GpuJob, ImageRecord
|
||||
from backend.app.services.ml.gpu_jobs import GpuJobService
|
||||
from backend.app.services.ml.regions import RegionService
|
||||
|
||||
@@ -148,3 +149,39 @@ async def test_release_hands_job_back_to_pending(client, db):
|
||||
assert resp.status_code == 200 and (await resp.get_json())["released"] == 1
|
||||
st = await (await client.get("/api/gpu/status")).get_json()
|
||||
assert st["pending"] == 1 and st["leased"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_errors_requeues_only_errored(client, db):
|
||||
"""/retry_errors resets ERRORED jobs (any task) to pending with a fresh
|
||||
retry budget — and leaves done work untouched (it is NOT /reprocess)."""
|
||||
img1 = await _img(db, "1" * 64)
|
||||
img2 = await _img(db, "2" * 64)
|
||||
svc = GpuJobService(db)
|
||||
j_err = await svc.enqueue(img1.id, "ccip")
|
||||
j_done = await svc.enqueue(img2.id, "siglip")
|
||||
err_id = j_err.id
|
||||
done_id = j_done.id
|
||||
j_err.status = "error"
|
||||
j_err.attempts = 3
|
||||
j_err.error = "no frames sampled from video (unprocessable)"
|
||||
j_done.status = "done"
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post("/api/gpu/retry_errors")
|
||||
assert resp.status_code == 200
|
||||
assert (await resp.get_json())["requeued"] == 1
|
||||
|
||||
# Column selects, not ORM refresh — the route wrote via Core DML.
|
||||
row = (await db.execute(
|
||||
select(GpuJob.status, GpuJob.attempts, GpuJob.error)
|
||||
.where(GpuJob.id == err_id)
|
||||
)).one()
|
||||
assert tuple(row) == ("pending", 0, None)
|
||||
done_status = await db.scalar(
|
||||
select(GpuJob.status).where(GpuJob.id == done_id)
|
||||
)
|
||||
assert done_status == "done"
|
||||
|
||||
st = await (await client.get("/api/gpu/status")).get_json()
|
||||
assert st["pending"] == 1 and st["error"] == 0
|
||||
|
||||
Reference in New Issue
Block a user