fix(gpu-jobs): end the error-tombstone loop — deliberate retry semantics + poison-job guards
The hourly ccip backfill's skip-list lacked 'error' (and the daily
siglip/embed variants re-gated failures on their missing results), so every
permanently-bad file got a fresh doomed job each run — ~24 duplicate error
rows/day per file, the perpetual 'unprocessable' flood. An errored job is now
a TOMBSTONE: no backfill re-enqueues it; retry is deliberate-only via
/retry_errors (an errored back-catalogue needs one button press after a
model swap).
One shared set of dedupe DELETEs (services/ml/gpu_jobs.error_dedupe_statements)
runs before every backfill and inside /retry_errors: error rows made moot by a
later pending/leased/done row go first, then older duplicates (newest reason
survives) — so the error count reads as distinct failing files and a retry
can't fan one file out into duplicate pending jobs. /retry_errors now returns
{requeued, pruned} and the toast shows both.
Poison-loop guards (release and lease-expiry burn no attempts, so a job that
stalls its transfer or crashes the agent every time cycled forever —
operator-observed jobs 99044/125288/131594/143131):
- agent: 3 in-session transient bounces (fetch or submit) → fail with the real
reason instead of another release; strikes never count while stopping, and
clear on submit success. Agent build 2026-07-02.3.
- server: the 60s orphan sweep (statements shared between the beat task and
GpuJobService so they can't drift) converts expired leases with >=5 lease
grants and pending jobs with >=10 to 'error', preserving the last stored
failure reason. Backstops old agent builds.
Tests: tombstone rule across all three backfill variants, moot-row pruning,
poison conversions, and the extended /retry_errors dedupe contract.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
+45
-25
@@ -458,15 +458,30 @@ def enqueue_gpu_backfill(task_name: str) -> int:
|
||||
|
||||
'siglip' gates on the RESULT (no concept region yet) rather than on a prior
|
||||
job, so it picks up the back-catalogue of images that were CCIP-embedded
|
||||
before concept crops existed, and retries images whose concept embed failed —
|
||||
without re-touching their figure/CCIP regions."""
|
||||
before concept crops existed — without re-touching their figure/CCIP regions.
|
||||
|
||||
An ERRORED job is a tombstone for its (image, task): no variant re-enqueues
|
||||
it. Retry is deliberate-only (/retry_errors), which also means an errored
|
||||
back-catalogue needs one "Retry errored jobs" press after a model swap.
|
||||
Before the tombstone rule, this loop re-minted a fresh doomed job for every
|
||||
permanently-bad file each run — ~24 duplicate error rows/day per file (the
|
||||
2026-07-02 "unprocessable" flood)."""
|
||||
from sqlalchemy import exists, insert, literal, or_
|
||||
from sqlalchemy import select as sa_select
|
||||
|
||||
from ..models import GpuJob, ImageRecord, ImageRegion, MLSettings
|
||||
from ..services.ml.gpu_jobs import error_dedupe_statements
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
# Prune stale tombstones first (loop-era duplicates + rows made moot by
|
||||
# a later success), so 'error' reads as one row per distinct failing
|
||||
# file and the skip-guards below see a clean picture.
|
||||
pruned = sum(
|
||||
session.execute(s).rowcount or 0 for s in error_dedupe_statements()
|
||||
)
|
||||
if pruned:
|
||||
log.info("gpu backfill: pruned %d stale/duplicate error rows", pruned)
|
||||
cur_version = session.execute(
|
||||
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||
).scalar_one()
|
||||
@@ -478,14 +493,15 @@ def enqueue_gpu_backfill(task_name: str) -> int:
|
||||
ImageRecord.siglip_model_version.is_(None),
|
||||
ImageRecord.siglip_model_version != cur_version,
|
||||
)
|
||||
queued = exists().where(
|
||||
# 'error' blocks too — tombstone rule, see docstring.
|
||||
blocked = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == "embed",
|
||||
GpuJob.status.in_(["pending", "leased"]),
|
||||
GpuJob.status.in_(["pending", "leased", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal("embed"), literal("pending")
|
||||
).where(stale).where(~queued)
|
||||
).where(stale).where(~blocked)
|
||||
elif task_name == "siglip":
|
||||
# Concept-crop re-embed: enqueue when there's no concept region AT THE
|
||||
# CURRENT model version — so a model swap re-triggers crops too, not
|
||||
@@ -495,19 +511,22 @@ def enqueue_gpu_backfill(task_name: str) -> int:
|
||||
ImageRegion.kind == "concept",
|
||||
ImageRegion.embedding_version == cur_version,
|
||||
)
|
||||
queued = exists().where(
|
||||
# 'error' blocks too — tombstone rule, see docstring.
|
||||
blocked = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == "siglip",
|
||||
GpuJob.status.in_(["pending", "leased"]),
|
||||
GpuJob.status.in_(["pending", "leased", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal("siglip"), literal("pending")
|
||||
).where(~has_current_concept).where(~queued)
|
||||
).where(~has_current_concept).where(~blocked)
|
||||
else:
|
||||
# ANY prior row blocks — including 'error' (tombstone rule, see
|
||||
# docstring): pre-fix this branch ran HOURLY and was the loop.
|
||||
already = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == task_name,
|
||||
GpuJob.status.in_(["pending", "leased", "done"]),
|
||||
GpuJob.status.in_(["pending", "leased", "done", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal(task_name), literal("pending")
|
||||
@@ -525,28 +544,29 @@ def enqueue_gpu_backfill(task_name: str) -> int:
|
||||
@celery.task(name="backend.app.tasks.ml.recover_orphaned_gpu_jobs")
|
||||
def recover_orphaned_gpu_jobs() -> int:
|
||||
"""Reset expired GPU-job leases back to pending — recovers work orphaned by an
|
||||
agent that died mid-job (no graceful release). Short beat cadence so orphans
|
||||
get picked back up quickly + the queue counts read honestly. Returns the
|
||||
number recovered."""
|
||||
agent that died mid-job (no graceful release) — and convert poison-loopers
|
||||
(release/expiry cycles that never reach fail()'s attempt cap) to 'error'.
|
||||
Statements are shared with GpuJobService.recover_orphaned so the sweep and
|
||||
the service can't drift. Short beat cadence so orphans get picked back up
|
||||
quickly + the queue counts read honestly. Returns the number recovered."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import update
|
||||
|
||||
from ..models import GpuJob
|
||||
from ..services.ml.gpu_jobs import recover_statements
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
now = datetime.now(UTC)
|
||||
res = session.execute(
|
||||
update(GpuJob)
|
||||
.where(GpuJob.status == "leased", GpuJob.lease_expires_at < now)
|
||||
.values(
|
||||
status="pending", lease_token=None, leased_at=None,
|
||||
lease_expires_at=None, updated_at=now,
|
||||
)
|
||||
)
|
||||
counts = {
|
||||
name: session.execute(stmt).rowcount or 0
|
||||
for name, stmt in recover_statements(datetime.now(UTC)).items()
|
||||
}
|
||||
session.commit()
|
||||
return res.rowcount or 0
|
||||
if counts["poison_expired"] or counts["poison_pending"]:
|
||||
log.warning(
|
||||
"gpu jobs poisoned -> error: %d crash-loop (expired lease), "
|
||||
"%d never-complete (pending)",
|
||||
counts["poison_expired"], counts["poison_pending"],
|
||||
)
|
||||
return counts["recovered"]
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.reprocess_gpu_jobs")
|
||||
|
||||
Reference in New Issue
Block a user