feat(triage): failed-processing triage — probe errored files, flag defects, recover (#125 C1-C3)
An errored GPU job's stored reason is a suspicion; the file probe is the
verdict. A 15-min beat sweep (triage_gpu_errors) runs verify_integrity's own
probe (sha256 + decode) on each errored image ONCE and writes both verdicts:
ImageRecord.integrity_status and the new GpuJob.triage_status ('defect' |
'file_ok', migration 0072). Every classification logs at WARNING so it
surfaces in Logs/System Activity.
- 'defect' rows are excluded from /retry_errors (re-running a known-bad file
burns agent time re-minting the tombstone); response now reports
defects_kept and the GpuAgentCard toast says so.
- GET /api/gpu/errors: triage view — reason buckets (classify_reason),
probe verdicts, per-job detail. POST /errors/triage runs the sweep now.
- POST /api/gpu/errors/<id>/recover: reuses the Layer-2 refetch pattern —
delete the defective copy + record (full cascade takes the tombstones too)
and re-poll its subscription Source so a fresh copy re-imports and re-enters
the pipeline; 'no_source' when nothing pollable resolves.
- New 'Failed processing' card (GpuTriageCard) in Maintenance: verdict counts,
reason summary, probe-now, defect list with thumbnails + per-image Recover.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
+112
-6
@@ -9,19 +9,25 @@ homelab admin.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
|
||||
from ..services.gallery_service import image_url
|
||||
from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
|
||||
from ..services.ml.gpu_triage import classify_reason, recover_defective_image
|
||||
from ..services.ml.regions import RegionService
|
||||
|
||||
gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
|
||||
|
||||
# Same container mount the maintenance tasks use (tasks/admin.py) — recovery
|
||||
# deletes the defective original + thumbnail under it.
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
_TOKEN_KEY = "gpu_agent_token"
|
||||
|
||||
|
||||
@@ -118,22 +124,122 @@ async def retry_errors():
|
||||
fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
|
||||
rows a later success made moot — the same statements the backfills run), so
|
||||
one failing file requeues as ONE job, never a fan-out of duplicates. Small
|
||||
row count (errors only) → inline statements; the response carries both
|
||||
counts for the UI toast."""
|
||||
row count (errors only) → inline statements; the response carries the
|
||||
counts for the UI toast. Triage-confirmed defects are NOT requeued (see
|
||||
the WHERE below) — they stay on the recovery surface."""
|
||||
async with get_session() as session:
|
||||
pruned = 0
|
||||
for stmt in error_dedupe_statements():
|
||||
pruned += (await session.execute(stmt)).rowcount or 0
|
||||
res = await session.execute(
|
||||
update(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
.where(
|
||||
GpuJob.status == "error",
|
||||
# Triage-confirmed DEFECTS stay errored: the integrity probe
|
||||
# already proved the FILE is bad, so re-running the job just
|
||||
# burns agent time re-minting the same tombstone — those go
|
||||
# through /errors/<id>/recover instead.
|
||||
or_(GpuJob.triage_status.is_(None),
|
||||
GpuJob.triage_status != "defect"),
|
||||
)
|
||||
.values(
|
||||
status="pending", attempts=0, error=None, lease_token=None,
|
||||
leased_at=None, lease_expires_at=None, updated_at=func.now(),
|
||||
leased_at=None, lease_expires_at=None, triage_status=None,
|
||||
updated_at=func.now(),
|
||||
)
|
||||
)
|
||||
kept = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
await session.commit()
|
||||
return jsonify({"requeued": res.rowcount or 0, "pruned": pruned})
|
||||
return jsonify({
|
||||
"requeued": res.rowcount or 0, "pruned": pruned, "defects_kept": kept,
|
||||
})
|
||||
|
||||
|
||||
# --- Failure triage + recovery (#125) ------------------------------------
|
||||
|
||||
@gpu_bp.route("/errors", methods=["GET"])
|
||||
async def errors():
|
||||
"""The triage view of the error tombstones: every errored job joined with
|
||||
its image's integrity verdict, bucketed by reason for the overview. The
|
||||
probe sweep (triage_gpu_errors, 15-min beat) fills triage_status; 'defect'
|
||||
rows are the recovery surface's list."""
|
||||
async with get_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
GpuJob.id, GpuJob.image_record_id, GpuJob.task,
|
||||
GpuJob.error, GpuJob.triage_status, GpuJob.updated_at,
|
||||
ImageRecord.integrity_status, ImageRecord.mime,
|
||||
ImageRecord.path, ImageRecord.thumbnail_path,
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.id == GpuJob.image_record_id)
|
||||
.where(GpuJob.status == "error")
|
||||
.order_by(GpuJob.updated_at.desc())
|
||||
.limit(500)
|
||||
)
|
||||
).all()
|
||||
total = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
by_class: dict[str, int] = {}
|
||||
triage = {"defect": 0, "file_ok": 0, "unclassified": 0}
|
||||
items = []
|
||||
for r in rows:
|
||||
cls = classify_reason(r.error)
|
||||
by_class[cls] = by_class.get(cls, 0) + 1
|
||||
bucket = r.triage_status or "unclassified"
|
||||
triage[bucket] = triage.get(bucket, 0) + 1
|
||||
items.append({
|
||||
"job_id": r.id,
|
||||
"image_id": r.image_record_id,
|
||||
"task": r.task,
|
||||
"error": r.error,
|
||||
"reason_class": cls,
|
||||
"triage_status": r.triage_status,
|
||||
"integrity_status": r.integrity_status,
|
||||
"mime": r.mime,
|
||||
"image_url": image_url(r.path),
|
||||
"thumbnail_url": (
|
||||
image_url(r.thumbnail_path) if r.thumbnail_path else None
|
||||
),
|
||||
"updated_at": r.updated_at.isoformat() if r.updated_at else None,
|
||||
})
|
||||
return jsonify({
|
||||
"total": total, "by_class": by_class, "triage": triage, "items": items,
|
||||
})
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/triage", methods=["POST"])
|
||||
async def errors_triage():
|
||||
"""Run the probe sweep NOW (the card's button) instead of waiting out the
|
||||
15-minute beat cadence."""
|
||||
from ..tasks.maintenance import triage_gpu_errors
|
||||
|
||||
r = triage_gpu_errors.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/<int:image_id>/recover", methods=["POST"])
|
||||
async def errors_recover(image_id: int):
|
||||
"""Recover a defect-triaged original: delete the bad copy + record and
|
||||
re-poll its subscription Source (a fresh fetch re-imports the file, which
|
||||
re-enters the GPU pipeline). Returns status 'no_source' when nothing
|
||||
pollable resolves — the file needs manual replacement there."""
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: recover_defective_image(
|
||||
s, image_id, images_root=_IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
||||
|
||||
Reference in New Issue
Block a user