feat(triage): failed-processing triage — probe errored files, flag defects, recover (#125 C1-C3)
An errored GPU job's stored reason is a suspicion; the file probe is the
verdict. A 15-min beat sweep (triage_gpu_errors) runs verify_integrity's own
probe (sha256 + decode) on each errored image ONCE and writes both verdicts:
ImageRecord.integrity_status and the new GpuJob.triage_status ('defect' |
'file_ok', migration 0072). Every classification logs at WARNING so it
surfaces in Logs/System Activity.
- 'defect' rows are excluded from /retry_errors (re-running a known-bad file
burns agent time re-minting the tombstone); response now reports
defects_kept and the GpuAgentCard toast says so.
- GET /api/gpu/errors: triage view — reason buckets (classify_reason),
probe verdicts, per-job detail. POST /errors/triage runs the sweep now.
- POST /api/gpu/errors/<id>/recover: reuses the Layer-2 refetch pattern —
delete the defective copy + record (full cascade takes the tombstones too)
and re-poll its subscription Source so a fresh copy re-imports and re-enters
the pipeline; 'no_source' when nothing pollable resolves.
- New 'Failed processing' card (GpuTriageCard) in Maintenance: verdict counts,
reason summary, probe-now, defect list with thumbnails + per-image Recover.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
+112
-6
@@ -9,19 +9,25 @@ homelab admin.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
|
||||
from ..services.gallery_service import image_url
|
||||
from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
|
||||
from ..services.ml.gpu_triage import classify_reason, recover_defective_image
|
||||
from ..services.ml.regions import RegionService
|
||||
|
||||
gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
|
||||
|
||||
# Same container mount the maintenance tasks use (tasks/admin.py) — recovery
|
||||
# deletes the defective original + thumbnail under it.
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
_TOKEN_KEY = "gpu_agent_token"
|
||||
|
||||
|
||||
@@ -118,22 +124,122 @@ async def retry_errors():
|
||||
fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
|
||||
rows a later success made moot — the same statements the backfills run), so
|
||||
one failing file requeues as ONE job, never a fan-out of duplicates. Small
|
||||
row count (errors only) → inline statements; the response carries both
|
||||
counts for the UI toast."""
|
||||
row count (errors only) → inline statements; the response carries the
|
||||
counts for the UI toast. Triage-confirmed defects are NOT requeued (see
|
||||
the WHERE below) — they stay on the recovery surface."""
|
||||
async with get_session() as session:
|
||||
pruned = 0
|
||||
for stmt in error_dedupe_statements():
|
||||
pruned += (await session.execute(stmt)).rowcount or 0
|
||||
res = await session.execute(
|
||||
update(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
.where(
|
||||
GpuJob.status == "error",
|
||||
# Triage-confirmed DEFECTS stay errored: the integrity probe
|
||||
# already proved the FILE is bad, so re-running the job just
|
||||
# burns agent time re-minting the same tombstone — those go
|
||||
# through /errors/<id>/recover instead.
|
||||
or_(GpuJob.triage_status.is_(None),
|
||||
GpuJob.triage_status != "defect"),
|
||||
)
|
||||
.values(
|
||||
status="pending", attempts=0, error=None, lease_token=None,
|
||||
leased_at=None, lease_expires_at=None, updated_at=func.now(),
|
||||
leased_at=None, lease_expires_at=None, triage_status=None,
|
||||
updated_at=func.now(),
|
||||
)
|
||||
)
|
||||
kept = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
await session.commit()
|
||||
return jsonify({"requeued": res.rowcount or 0, "pruned": pruned})
|
||||
return jsonify({
|
||||
"requeued": res.rowcount or 0, "pruned": pruned, "defects_kept": kept,
|
||||
})
|
||||
|
||||
|
||||
# --- Failure triage + recovery (#125) ------------------------------------
|
||||
|
||||
@gpu_bp.route("/errors", methods=["GET"])
|
||||
async def errors():
|
||||
"""The triage view of the error tombstones: every errored job joined with
|
||||
its image's integrity verdict, bucketed by reason for the overview. The
|
||||
probe sweep (triage_gpu_errors, 15-min beat) fills triage_status; 'defect'
|
||||
rows are the recovery surface's list."""
|
||||
async with get_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
GpuJob.id, GpuJob.image_record_id, GpuJob.task,
|
||||
GpuJob.error, GpuJob.triage_status, GpuJob.updated_at,
|
||||
ImageRecord.integrity_status, ImageRecord.mime,
|
||||
ImageRecord.path, ImageRecord.thumbnail_path,
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.id == GpuJob.image_record_id)
|
||||
.where(GpuJob.status == "error")
|
||||
.order_by(GpuJob.updated_at.desc())
|
||||
.limit(500)
|
||||
)
|
||||
).all()
|
||||
total = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
by_class: dict[str, int] = {}
|
||||
triage = {"defect": 0, "file_ok": 0, "unclassified": 0}
|
||||
items = []
|
||||
for r in rows:
|
||||
cls = classify_reason(r.error)
|
||||
by_class[cls] = by_class.get(cls, 0) + 1
|
||||
bucket = r.triage_status or "unclassified"
|
||||
triage[bucket] = triage.get(bucket, 0) + 1
|
||||
items.append({
|
||||
"job_id": r.id,
|
||||
"image_id": r.image_record_id,
|
||||
"task": r.task,
|
||||
"error": r.error,
|
||||
"reason_class": cls,
|
||||
"triage_status": r.triage_status,
|
||||
"integrity_status": r.integrity_status,
|
||||
"mime": r.mime,
|
||||
"image_url": image_url(r.path),
|
||||
"thumbnail_url": (
|
||||
image_url(r.thumbnail_path) if r.thumbnail_path else None
|
||||
),
|
||||
"updated_at": r.updated_at.isoformat() if r.updated_at else None,
|
||||
})
|
||||
return jsonify({
|
||||
"total": total, "by_class": by_class, "triage": triage, "items": items,
|
||||
})
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/triage", methods=["POST"])
|
||||
async def errors_triage():
|
||||
"""Run the probe sweep NOW (the card's button) instead of waiting out the
|
||||
15-minute beat cadence."""
|
||||
from ..tasks.maintenance import triage_gpu_errors
|
||||
|
||||
r = triage_gpu_errors.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/<int:image_id>/recover", methods=["POST"])
|
||||
async def errors_recover(image_id: int):
|
||||
"""Recover a defect-triaged original: delete the bad copy + record and
|
||||
re-poll its subscription Source (a fresh fetch re-imports the file, which
|
||||
re-enters the GPU pipeline). Returns status 'no_source' when nothing
|
||||
pollable resolves — the file needs manual replacement there."""
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: recover_defective_image(
|
||||
s, image_id, images_root=_IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
||||
|
||||
@@ -109,6 +109,10 @@ def make_celery() -> Celery:
|
||||
"task": "backend.app.tasks.ml.recover_orphaned_gpu_jobs",
|
||||
"schedule": 60.0, # quick pickup of work a dead agent orphaned
|
||||
},
|
||||
"triage-gpu-errors": {
|
||||
"task": "backend.app.tasks.maintenance.triage_gpu_errors",
|
||||
"schedule": 900.0, # probe errored jobs' files → defect/file_ok
|
||||
},
|
||||
"enqueue-ccip-backfill-hourly": {
|
||||
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
||||
"schedule": 3600.0, # auto-feed NEW images; errored are
|
||||
|
||||
@@ -62,6 +62,11 @@ class GpuJob(Base):
|
||||
)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Triage verdict for an ERRORED job (#125): NULL = not yet probed;
|
||||
# 'defect' = the integrity probe says the FILE itself is bad (surfaced for
|
||||
# recovery, excluded from /retry_errors); 'file_ok' = the file passes —
|
||||
# the failure was operational (timeout/transient), safe to retry.
|
||||
triage_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""GPU-failure triage (#125): classify errored jobs, PROBE the file, recover.
|
||||
|
||||
An errored GPU job is a tombstone with a stored reason, but the reason alone is
|
||||
a suspicion, not a verdict — a timeout can hit a perfectly fine file, and
|
||||
"moov atom not found" can mean a truncated download OR a one-off transfer
|
||||
fault. So triage EVALUATES: it runs the real integrity probe (sha256 recompute
|
||||
+ PIL/ffprobe — verify_integrity's own machinery) on each errored image ONCE
|
||||
and records both verdicts:
|
||||
|
||||
ImageRecord.integrity_status <- file-level verdict (ok / corrupt / ...)
|
||||
GpuJob.triage_status <- 'defect' (file is bad: recovery material,
|
||||
excluded from /retry_errors)
|
||||
'file_ok' (file passes: the failure was
|
||||
operational, safe to retry)
|
||||
|
||||
Recovery reuses established primitives: delete the defective copy + record
|
||||
(cleanup_service.delete_images — full cascade) and re-poll the image's
|
||||
subscription Source (the Layer-2 refetch pattern: gallery-dl re-fetches the
|
||||
now-absent file on the next source check). Images without a pollable Source
|
||||
report 'no_source' — manual remediation. Every classification is logged at
|
||||
WARNING so the operator notices in Logs / System Activity.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...models import GpuJob, ImageProvenance, ImageRecord, Source
|
||||
from ..cleanup_service import delete_images
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Reason buckets for the triage overview (reporting only — the PROBE decides
|
||||
# 'defect', never the string). Ordered: first match wins.
|
||||
_REASON_BUCKETS = (
|
||||
("poisoned", ("poisoned:",)),
|
||||
("transient", ("gave up after repeated transient", "curator unreachable",
|
||||
"connection", "read timed out")),
|
||||
("timeout", ("timed out", "timeout")),
|
||||
("truncated_or_corrupt", ("moov atom", "invalid data", "end of file",
|
||||
"header missing", "error reading header",
|
||||
"truncated", "premature", "corrupt",
|
||||
"no frames sampled")),
|
||||
("decode", ("cannot identify", "decompression", "broken data stream",
|
||||
"unrecognized data")),
|
||||
)
|
||||
|
||||
|
||||
def classify_reason(error: str | None) -> str:
|
||||
"""Bucket a stored job-error string for the overview table."""
|
||||
text = (error or "").lower()
|
||||
if not text:
|
||||
return "other"
|
||||
for bucket, needles in _REASON_BUCKETS:
|
||||
if any(n in text for n in needles):
|
||||
return bucket
|
||||
return "other"
|
||||
|
||||
|
||||
def triage_errored_jobs(
|
||||
session: Session, *, time_budget_seconds: float = 300.0,
|
||||
) -> dict:
|
||||
"""Probe every not-yet-triaged errored image and write both verdicts.
|
||||
|
||||
Time-boxed (sha256 of a large original over NFS can take tens of seconds)
|
||||
and inherently resumable: rows are selected by `triage_status IS NULL`, so
|
||||
the next sweep continues exactly where a budget cut stopped. Commits per
|
||||
image so a mid-run crash keeps completed verdicts."""
|
||||
image_ids = session.execute(
|
||||
select(GpuJob.image_record_id)
|
||||
.where(GpuJob.status == "error", GpuJob.triage_status.is_(None))
|
||||
.group_by(GpuJob.image_record_id)
|
||||
.order_by(GpuJob.image_record_id)
|
||||
).scalars().all()
|
||||
counts = {"probed": 0, "defect": 0, "file_ok": 0, "partial": False}
|
||||
if not image_ids:
|
||||
return counts
|
||||
# Lazy imports: the probe helper lives in the maintenance task module and
|
||||
# the hasher in the importer — importing either at module load would pull
|
||||
# celery into every service consumer.
|
||||
from ...tasks.maintenance import _verify_one
|
||||
from ..importer import _sha256_of
|
||||
|
||||
started = time.monotonic()
|
||||
for image_id in image_ids:
|
||||
if time.monotonic() - started > time_budget_seconds:
|
||||
counts["partial"] = True
|
||||
break
|
||||
rec = session.get(ImageRecord, image_id)
|
||||
if rec is None: # record deleted since the job errored
|
||||
continue
|
||||
verdict = _verify_one(Path(rec.path), rec.sha256, rec.mime, _sha256_of)
|
||||
# 'ok' means the failure was operational; anything else (corrupt /
|
||||
# failed_verification = missing/unreadable) makes the file itself the
|
||||
# problem — recovery material.
|
||||
triage = "file_ok" if verdict == "ok" else "defect"
|
||||
reason = session.execute(
|
||||
select(GpuJob.error)
|
||||
.where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
rec.integrity_status = verdict
|
||||
session.execute(
|
||||
update(GpuJob)
|
||||
.where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
|
||||
.values(triage_status=triage, updated_at=datetime.now(UTC))
|
||||
)
|
||||
session.commit()
|
||||
counts["probed"] += 1
|
||||
counts[triage] += 1
|
||||
log.warning(
|
||||
"gpu triage: image %s (%s) job error %r -> integrity probe %r -> %s",
|
||||
image_id, rec.path, (reason or "")[:120], verdict, triage,
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def recover_defective_image(
|
||||
session: Session, image_id: int, *, images_root: Path,
|
||||
) -> dict:
|
||||
"""Delete the defective copy + record and re-poll its subscription Source.
|
||||
|
||||
Mirrors the Layer-2 import refetch: with the bad file gone, the source's
|
||||
next gallery-dl run re-fetches a fresh copy, which re-imports as a new
|
||||
record and re-enters the GPU pipeline. The record delete cascades the
|
||||
error tombstones with it. 'no_source' when no enabled, real-URL Source is
|
||||
reachable via the image's provenance — manual remediation there."""
|
||||
rec = session.get(ImageRecord, image_id)
|
||||
if rec is None:
|
||||
return {"status": "not_found"}
|
||||
src_id = session.execute(
|
||||
select(Source.id)
|
||||
.join(ImageProvenance, ImageProvenance.source_id == Source.id)
|
||||
.where(
|
||||
ImageProvenance.image_record_id == image_id,
|
||||
Source.enabled.is_(True),
|
||||
~Source.url.like("sidecar:%"), # synthetic anchor — not pollable
|
||||
)
|
||||
.order_by(Source.id.asc())
|
||||
).scalars().first()
|
||||
if src_id is None:
|
||||
return {"status": "no_source"}
|
||||
path = rec.path
|
||||
summary = delete_images(session, image_ids=[image_id], images_root=images_root)
|
||||
# Lazy import (services -> tasks would cycle at module load).
|
||||
from ...tasks.download import download_source
|
||||
|
||||
download_source.delay(src_id)
|
||||
log.warning(
|
||||
"gpu triage recovery: deleted defective image %s (%s) and queued a "
|
||||
"re-check of source %s to re-fetch it", image_id, path, src_id,
|
||||
)
|
||||
return {"status": "refetch_queued", "source_id": src_id, **summary}
|
||||
@@ -582,6 +582,28 @@ def verify_integrity() -> int:
|
||||
return total
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.maintenance.triage_gpu_errors",
|
||||
# Bounded small-set probe (only errored images, once each), but a single
|
||||
# large original's sha256 over NFS can run tens of seconds — same quick-lane
|
||||
# tolerance rationale as verify_integrity above.
|
||||
soft_time_limit=600, time_limit=900,
|
||||
)
|
||||
def triage_gpu_errors() -> dict:
|
||||
"""Failure triage (#125): probe each errored GPU job's file once and write
|
||||
the verdicts (ImageRecord.integrity_status + GpuJob.triage_status) — see
|
||||
services/ml/gpu_triage.py. Time-boxed + resumable; no-op when every errored
|
||||
job is already triaged."""
|
||||
from ..services.ml.gpu_triage import triage_errored_jobs
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
summary = triage_errored_jobs(session, time_budget_seconds=300.0)
|
||||
if summary["probed"]:
|
||||
log.info("triage_gpu_errors: %s", summary)
|
||||
return summary
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_download_events")
|
||||
def recover_stalled_download_events() -> int:
|
||||
"""Recover DownloadEvent rows stuck pending/running past the worker hard kill.
|
||||
|
||||
Reference in New Issue
Block a user