feat(triage): failed-processing triage — probe errored files, flag defects, recover (#125 C1-C3)
An errored GPU job's stored reason is a suspicion; the file probe is the
verdict. A 15-min beat sweep (triage_gpu_errors) runs verify_integrity's own
probe (sha256 + decode) on each errored image ONCE and writes both verdicts:
ImageRecord.integrity_status and the new GpuJob.triage_status ('defect' |
'file_ok', migration 0072). Every classification logs at WARNING so it
surfaces in Logs/System Activity.
- 'defect' rows are excluded from /retry_errors (re-running a known-bad file
burns agent time re-minting the tombstone); response now reports
defects_kept and the GpuAgentCard toast says so.
- GET /api/gpu/errors: triage view — reason buckets (classify_reason),
probe verdicts, per-job detail. POST /errors/triage runs the sweep now.
- POST /api/gpu/errors/<id>/recover: reuses the Layer-2 refetch pattern —
delete the defective copy + record (full cascade takes the tombstones too)
and re-poll its subscription Source so a fresh copy re-imports and re-enters
the pipeline; 'no_source' when nothing pollable resolves.
- New 'Failed processing' card (GpuTriageCard) in Maintenance: verdict counts,
reason summary, probe-now, defect list with thumbnails + per-image Recover.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
"""gpu_job.triage_status — the probe's verdict on an errored job's FILE
|
||||
|
||||
Failure triage (#125): a periodic sweep probes each errored image's file
|
||||
(sha256 + decode, verify_integrity's machinery) exactly once and stores the
|
||||
verdict here — 'defect' (the file is bad: recovery material, excluded from
|
||||
/retry_errors) or 'file_ok' (failure was operational, safe to retry). NULL
|
||||
means not yet probed; selecting on NULL is what makes the sweep resumable.
|
||||
No index: the errored slice the sweep scans is tiny by design (tombstones).
|
||||
|
||||
Revision ID: 0072
|
||||
Revises: 0071
|
||||
Create Date: 2026-07-02
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0072"
|
||||
down_revision: Union[str, None] = "0071"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"gpu_job", sa.Column("triage_status", sa.String(16), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("gpu_job", "triage_status")
|
||||
+112
-6
@@ -9,19 +9,25 @@ homelab admin.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
|
||||
from ..services.gallery_service import image_url
|
||||
from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
|
||||
from ..services.ml.gpu_triage import classify_reason, recover_defective_image
|
||||
from ..services.ml.regions import RegionService
|
||||
|
||||
gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
|
||||
|
||||
# Same container mount the maintenance tasks use (tasks/admin.py) — recovery
|
||||
# deletes the defective original + thumbnail under it.
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
_TOKEN_KEY = "gpu_agent_token"
|
||||
|
||||
|
||||
@@ -118,22 +124,122 @@ async def retry_errors():
|
||||
fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
|
||||
rows a later success made moot — the same statements the backfills run), so
|
||||
one failing file requeues as ONE job, never a fan-out of duplicates. Small
|
||||
row count (errors only) → inline statements; the response carries both
|
||||
counts for the UI toast."""
|
||||
row count (errors only) → inline statements; the response carries the
|
||||
counts for the UI toast. Triage-confirmed defects are NOT requeued (see
|
||||
the WHERE below) — they stay on the recovery surface."""
|
||||
async with get_session() as session:
|
||||
pruned = 0
|
||||
for stmt in error_dedupe_statements():
|
||||
pruned += (await session.execute(stmt)).rowcount or 0
|
||||
res = await session.execute(
|
||||
update(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
.where(
|
||||
GpuJob.status == "error",
|
||||
# Triage-confirmed DEFECTS stay errored: the integrity probe
|
||||
# already proved the FILE is bad, so re-running the job just
|
||||
# burns agent time re-minting the same tombstone — those go
|
||||
# through /errors/<id>/recover instead.
|
||||
or_(GpuJob.triage_status.is_(None),
|
||||
GpuJob.triage_status != "defect"),
|
||||
)
|
||||
.values(
|
||||
status="pending", attempts=0, error=None, lease_token=None,
|
||||
leased_at=None, lease_expires_at=None, updated_at=func.now(),
|
||||
leased_at=None, lease_expires_at=None, triage_status=None,
|
||||
updated_at=func.now(),
|
||||
)
|
||||
)
|
||||
kept = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
await session.commit()
|
||||
return jsonify({"requeued": res.rowcount or 0, "pruned": pruned})
|
||||
return jsonify({
|
||||
"requeued": res.rowcount or 0, "pruned": pruned, "defects_kept": kept,
|
||||
})
|
||||
|
||||
|
||||
# --- Failure triage + recovery (#125) ------------------------------------
|
||||
|
||||
@gpu_bp.route("/errors", methods=["GET"])
|
||||
async def errors():
|
||||
"""The triage view of the error tombstones: every errored job joined with
|
||||
its image's integrity verdict, bucketed by reason for the overview. The
|
||||
probe sweep (triage_gpu_errors, 15-min beat) fills triage_status; 'defect'
|
||||
rows are the recovery surface's list."""
|
||||
async with get_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
GpuJob.id, GpuJob.image_record_id, GpuJob.task,
|
||||
GpuJob.error, GpuJob.triage_status, GpuJob.updated_at,
|
||||
ImageRecord.integrity_status, ImageRecord.mime,
|
||||
ImageRecord.path, ImageRecord.thumbnail_path,
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.id == GpuJob.image_record_id)
|
||||
.where(GpuJob.status == "error")
|
||||
.order_by(GpuJob.updated_at.desc())
|
||||
.limit(500)
|
||||
)
|
||||
).all()
|
||||
total = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
by_class: dict[str, int] = {}
|
||||
triage = {"defect": 0, "file_ok": 0, "unclassified": 0}
|
||||
items = []
|
||||
for r in rows:
|
||||
cls = classify_reason(r.error)
|
||||
by_class[cls] = by_class.get(cls, 0) + 1
|
||||
bucket = r.triage_status or "unclassified"
|
||||
triage[bucket] = triage.get(bucket, 0) + 1
|
||||
items.append({
|
||||
"job_id": r.id,
|
||||
"image_id": r.image_record_id,
|
||||
"task": r.task,
|
||||
"error": r.error,
|
||||
"reason_class": cls,
|
||||
"triage_status": r.triage_status,
|
||||
"integrity_status": r.integrity_status,
|
||||
"mime": r.mime,
|
||||
"image_url": image_url(r.path),
|
||||
"thumbnail_url": (
|
||||
image_url(r.thumbnail_path) if r.thumbnail_path else None
|
||||
),
|
||||
"updated_at": r.updated_at.isoformat() if r.updated_at else None,
|
||||
})
|
||||
return jsonify({
|
||||
"total": total, "by_class": by_class, "triage": triage, "items": items,
|
||||
})
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/triage", methods=["POST"])
|
||||
async def errors_triage():
|
||||
"""Run the probe sweep NOW (the card's button) instead of waiting out the
|
||||
15-minute beat cadence."""
|
||||
from ..tasks.maintenance import triage_gpu_errors
|
||||
|
||||
r = triage_gpu_errors.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/<int:image_id>/recover", methods=["POST"])
|
||||
async def errors_recover(image_id: int):
|
||||
"""Recover a defect-triaged original: delete the bad copy + record and
|
||||
re-poll its subscription Source (a fresh fetch re-imports the file, which
|
||||
re-enters the GPU pipeline). Returns status 'no_source' when nothing
|
||||
pollable resolves — the file needs manual replacement there."""
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: recover_defective_image(
|
||||
s, image_id, images_root=_IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
||||
|
||||
@@ -109,6 +109,10 @@ def make_celery() -> Celery:
|
||||
"task": "backend.app.tasks.ml.recover_orphaned_gpu_jobs",
|
||||
"schedule": 60.0, # quick pickup of work a dead agent orphaned
|
||||
},
|
||||
"triage-gpu-errors": {
|
||||
"task": "backend.app.tasks.maintenance.triage_gpu_errors",
|
||||
"schedule": 900.0, # probe errored jobs' files → defect/file_ok
|
||||
},
|
||||
"enqueue-ccip-backfill-hourly": {
|
||||
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
||||
"schedule": 3600.0, # auto-feed NEW images; errored are
|
||||
|
||||
@@ -62,6 +62,11 @@ class GpuJob(Base):
|
||||
)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Triage verdict for an ERRORED job (#125): NULL = not yet probed;
|
||||
# 'defect' = the integrity probe says the FILE itself is bad (surfaced for
|
||||
# recovery, excluded from /retry_errors); 'file_ok' = the file passes —
|
||||
# the failure was operational (timeout/transient), safe to retry.
|
||||
triage_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""GPU-failure triage (#125): classify errored jobs, PROBE the file, recover.
|
||||
|
||||
An errored GPU job is a tombstone with a stored reason, but the reason alone is
|
||||
a suspicion, not a verdict — a timeout can hit a perfectly fine file, and
|
||||
"moov atom not found" can mean a truncated download OR a one-off transfer
|
||||
fault. So triage EVALUATES: it runs the real integrity probe (sha256 recompute
|
||||
+ PIL/ffprobe — verify_integrity's own machinery) on each errored image ONCE
|
||||
and records both verdicts:
|
||||
|
||||
ImageRecord.integrity_status <- file-level verdict (ok / corrupt / ...)
|
||||
GpuJob.triage_status <- 'defect' (file is bad: recovery material,
|
||||
excluded from /retry_errors)
|
||||
'file_ok' (file passes: the failure was
|
||||
operational, safe to retry)
|
||||
|
||||
Recovery reuses established primitives: delete the defective copy + record
|
||||
(cleanup_service.delete_images — full cascade) and re-poll the image's
|
||||
subscription Source (the Layer-2 refetch pattern: gallery-dl re-fetches the
|
||||
now-absent file on the next source check). Images without a pollable Source
|
||||
report 'no_source' — manual remediation. Every classification is logged at
|
||||
WARNING so the operator notices in Logs / System Activity.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...models import GpuJob, ImageProvenance, ImageRecord, Source
|
||||
from ..cleanup_service import delete_images
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Reason buckets for the triage overview (reporting only — the PROBE decides
|
||||
# 'defect', never the string). Ordered: first match wins.
|
||||
_REASON_BUCKETS = (
|
||||
("poisoned", ("poisoned:",)),
|
||||
("transient", ("gave up after repeated transient", "curator unreachable",
|
||||
"connection", "read timed out")),
|
||||
("timeout", ("timed out", "timeout")),
|
||||
("truncated_or_corrupt", ("moov atom", "invalid data", "end of file",
|
||||
"header missing", "error reading header",
|
||||
"truncated", "premature", "corrupt",
|
||||
"no frames sampled")),
|
||||
("decode", ("cannot identify", "decompression", "broken data stream",
|
||||
"unrecognized data")),
|
||||
)
|
||||
|
||||
|
||||
def classify_reason(error: str | None) -> str:
|
||||
"""Bucket a stored job-error string for the overview table."""
|
||||
text = (error or "").lower()
|
||||
if not text:
|
||||
return "other"
|
||||
for bucket, needles in _REASON_BUCKETS:
|
||||
if any(n in text for n in needles):
|
||||
return bucket
|
||||
return "other"
|
||||
|
||||
|
||||
def triage_errored_jobs(
|
||||
session: Session, *, time_budget_seconds: float = 300.0,
|
||||
) -> dict:
|
||||
"""Probe every not-yet-triaged errored image and write both verdicts.
|
||||
|
||||
Time-boxed (sha256 of a large original over NFS can take tens of seconds)
|
||||
and inherently resumable: rows are selected by `triage_status IS NULL`, so
|
||||
the next sweep continues exactly where a budget cut stopped. Commits per
|
||||
image so a mid-run crash keeps completed verdicts."""
|
||||
image_ids = session.execute(
|
||||
select(GpuJob.image_record_id)
|
||||
.where(GpuJob.status == "error", GpuJob.triage_status.is_(None))
|
||||
.group_by(GpuJob.image_record_id)
|
||||
.order_by(GpuJob.image_record_id)
|
||||
).scalars().all()
|
||||
counts = {"probed": 0, "defect": 0, "file_ok": 0, "partial": False}
|
||||
if not image_ids:
|
||||
return counts
|
||||
# Lazy imports: the probe helper lives in the maintenance task module and
|
||||
# the hasher in the importer — importing either at module load would pull
|
||||
# celery into every service consumer.
|
||||
from ...tasks.maintenance import _verify_one
|
||||
from ..importer import _sha256_of
|
||||
|
||||
started = time.monotonic()
|
||||
for image_id in image_ids:
|
||||
if time.monotonic() - started > time_budget_seconds:
|
||||
counts["partial"] = True
|
||||
break
|
||||
rec = session.get(ImageRecord, image_id)
|
||||
if rec is None: # record deleted since the job errored
|
||||
continue
|
||||
verdict = _verify_one(Path(rec.path), rec.sha256, rec.mime, _sha256_of)
|
||||
# 'ok' means the failure was operational; anything else (corrupt /
|
||||
# failed_verification = missing/unreadable) makes the file itself the
|
||||
# problem — recovery material.
|
||||
triage = "file_ok" if verdict == "ok" else "defect"
|
||||
reason = session.execute(
|
||||
select(GpuJob.error)
|
||||
.where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
rec.integrity_status = verdict
|
||||
session.execute(
|
||||
update(GpuJob)
|
||||
.where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
|
||||
.values(triage_status=triage, updated_at=datetime.now(UTC))
|
||||
)
|
||||
session.commit()
|
||||
counts["probed"] += 1
|
||||
counts[triage] += 1
|
||||
log.warning(
|
||||
"gpu triage: image %s (%s) job error %r -> integrity probe %r -> %s",
|
||||
image_id, rec.path, (reason or "")[:120], verdict, triage,
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def recover_defective_image(
|
||||
session: Session, image_id: int, *, images_root: Path,
|
||||
) -> dict:
|
||||
"""Delete the defective copy + record and re-poll its subscription Source.
|
||||
|
||||
Mirrors the Layer-2 import refetch: with the bad file gone, the source's
|
||||
next gallery-dl run re-fetches a fresh copy, which re-imports as a new
|
||||
record and re-enters the GPU pipeline. The record delete cascades the
|
||||
error tombstones with it. 'no_source' when no enabled, real-URL Source is
|
||||
reachable via the image's provenance — manual remediation there."""
|
||||
rec = session.get(ImageRecord, image_id)
|
||||
if rec is None:
|
||||
return {"status": "not_found"}
|
||||
src_id = session.execute(
|
||||
select(Source.id)
|
||||
.join(ImageProvenance, ImageProvenance.source_id == Source.id)
|
||||
.where(
|
||||
ImageProvenance.image_record_id == image_id,
|
||||
Source.enabled.is_(True),
|
||||
~Source.url.like("sidecar:%"), # synthetic anchor — not pollable
|
||||
)
|
||||
.order_by(Source.id.asc())
|
||||
).scalars().first()
|
||||
if src_id is None:
|
||||
return {"status": "no_source"}
|
||||
path = rec.path
|
||||
summary = delete_images(session, image_ids=[image_id], images_root=images_root)
|
||||
# Lazy import (services -> tasks would cycle at module load).
|
||||
from ...tasks.download import download_source
|
||||
|
||||
download_source.delay(src_id)
|
||||
log.warning(
|
||||
"gpu triage recovery: deleted defective image %s (%s) and queued a "
|
||||
"re-check of source %s to re-fetch it", image_id, path, src_id,
|
||||
)
|
||||
return {"status": "refetch_queued", "source_id": src_id, **summary}
|
||||
@@ -582,6 +582,28 @@ def verify_integrity() -> int:
|
||||
return total
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.maintenance.triage_gpu_errors",
|
||||
# Bounded small-set probe (only errored images, once each), but a single
|
||||
# large original's sha256 over NFS can run tens of seconds — same quick-lane
|
||||
# tolerance rationale as verify_integrity above.
|
||||
soft_time_limit=600, time_limit=900,
|
||||
)
|
||||
def triage_gpu_errors() -> dict:
|
||||
"""Failure triage (#125): probe each errored GPU job's file once and write
|
||||
the verdicts (ImageRecord.integrity_status + GpuJob.triage_status) — see
|
||||
services/ml/gpu_triage.py. Time-boxed + resumable; no-op when every errored
|
||||
job is already triaged."""
|
||||
from ..services.ml.gpu_triage import triage_errored_jobs
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
summary = triage_errored_jobs(session, time_budget_seconds=300.0)
|
||||
if summary["probed"]:
|
||||
log.info("triage_gpu_errors: %s", summary)
|
||||
return summary
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_download_events")
|
||||
def recover_stalled_download_events() -> int:
|
||||
"""Recover DownloadEvent rows stuck pending/running past the worker hard kill.
|
||||
|
||||
@@ -337,8 +337,11 @@ async function onBackfillSiglip() {
|
||||
async function onRetryErrors() {
|
||||
retrying.value = true
|
||||
try {
|
||||
const { requeued, pruned } = await store.retryErrors()
|
||||
const extra = pruned ? ` (${pruned} stale duplicate${pruned === 1 ? '' : 's'} pruned)` : ''
|
||||
const { requeued, pruned, defects_kept: kept } = await store.retryErrors()
|
||||
const extras = []
|
||||
if (pruned) extras.push(`${pruned} stale duplicate${pruned === 1 ? '' : 's'} pruned`)
|
||||
if (kept) extras.push(`${kept} known-bad file${kept === 1 ? '' : 's'} kept for recovery`)
|
||||
const extra = extras.length ? ` (${extras.join(', ')})` : ''
|
||||
toast({ text: `Requeued ${requeued} errored job${requeued === 1 ? '' : 's'}${extra} — run the agent to process them`, type: 'success' })
|
||||
await refreshQueue()
|
||||
} catch (e) {
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
<template>
|
||||
<MaintenanceTile
|
||||
icon="mdi-file-alert"
|
||||
title="Failed processing"
|
||||
blurb="Triage originals that failed GPU processing — probe the files, flag defects, recover them."
|
||||
>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
A job that keeps failing parks as an error with its reason. A background
|
||||
probe then checks the FILE itself (checksum + decode) and splits the
|
||||
errors: <b>defective files</b> (truncated/corrupt originals — listed below
|
||||
for recovery) vs <b>file OK</b> (the failure was operational; requeue
|
||||
those with <i>Retry errored jobs</i> on the GPU agent card).
|
||||
</p>
|
||||
|
||||
<div v-if="loading" class="fc-muted text-body-2">Loading…</div>
|
||||
<template v-else-if="overview">
|
||||
<div class="fc-queue mb-3">
|
||||
<div class="fc-q"><div class="fc-q__n">{{ overview.total }}</div><div class="fc-q__l">errored</div></div>
|
||||
<div class="fc-q"><div class="fc-q__n" :class="overview.triage.defect ? 'fc-weak' : ''">{{ overview.triage.defect }}</div><div class="fc-q__l">defective</div></div>
|
||||
<div class="fc-q"><div class="fc-q__n fc-good">{{ overview.triage.file_ok }}</div><div class="fc-q__l">file ok</div></div>
|
||||
<div class="fc-q"><div class="fc-q__n">{{ overview.triage.unclassified }}</div><div class="fc-q__l">unprobed</div></div>
|
||||
</div>
|
||||
|
||||
<p v-if="classSummary" class="fc-muted text-caption mb-3">
|
||||
Reasons: {{ classSummary }}
|
||||
</p>
|
||||
|
||||
<div class="d-flex mb-4" style="gap:8px">
|
||||
<v-btn
|
||||
size="small" color="accent" variant="tonal" rounded="pill"
|
||||
prepend-icon="mdi-magnify-scan" :loading="probing"
|
||||
:disabled="!overview.triage.unclassified" @click="onProbe"
|
||||
>Probe unclassified now</v-btn>
|
||||
<v-btn
|
||||
size="small" variant="text" rounded="pill"
|
||||
prepend-icon="mdi-refresh" @click="refresh"
|
||||
>Refresh</v-btn>
|
||||
</div>
|
||||
|
||||
<template v-if="defects.length">
|
||||
<div class="fc-section-h mb-2">Defective originals</div>
|
||||
<div v-for="it in defects" :key="it.job_id" class="fc-defect mb-2">
|
||||
<a :href="it.image_url" target="_blank" rel="noopener" class="fc-defect__thumb">
|
||||
<img v-if="it.thumbnail_url" :src="it.thumbnail_url" alt="">
|
||||
<v-icon v-else icon="mdi-file-question" size="28" />
|
||||
</a>
|
||||
<div class="fc-defect__meta">
|
||||
<div class="text-body-2">
|
||||
image <b>{{ it.image_id }}</b> · {{ it.task }} ·
|
||||
<span class="fc-weak">{{ it.integrity_status }}</span>
|
||||
</div>
|
||||
<div class="fc-muted text-caption fc-defect__err" :title="it.error || ''">
|
||||
{{ it.error || 'no stored reason' }}
|
||||
</div>
|
||||
</div>
|
||||
<v-btn
|
||||
v-if="recovered[it.image_id] !== 'no_source'"
|
||||
size="small" color="accent" variant="tonal" rounded="pill"
|
||||
prepend-icon="mdi-cloud-download" :loading="recovering === it.image_id"
|
||||
@click="onRecover(it)"
|
||||
>Recover</v-btn>
|
||||
<span v-else class="fc-muted text-caption">
|
||||
no pollable source — replace the file manually
|
||||
</span>
|
||||
</div>
|
||||
<p class="fc-muted text-caption mt-2 mb-0">
|
||||
Recover deletes the bad copy (and its record) and re-checks its
|
||||
subscription source, so a fresh download re-imports it and re-enters
|
||||
processing. Files without a pollable source need manual replacement.
|
||||
</p>
|
||||
</template>
|
||||
<p v-else-if="!overview.total" class="fc-muted text-body-2 mb-0">
|
||||
No failed jobs — the pipeline is clean.
|
||||
</p>
|
||||
<p v-else-if="!overview.triage.unclassified" class="fc-muted text-body-2 mb-0">
|
||||
No defective files — every probed failure was operational
|
||||
(file OK). Requeue them from the GPU agent card.
|
||||
</p>
|
||||
</template>
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import { useGpuStore } from '../../stores/gpu.js'
|
||||
|
||||
const store = useGpuStore()
|
||||
const loading = ref(true)
|
||||
const overview = ref(null)
|
||||
const probing = ref(false)
|
||||
const recovering = ref(null)
|
||||
// image_id -> 'no_source' for rows recovery already declined; keeps the
|
||||
// verdict visible instead of a button that fails the same way again.
|
||||
const recovered = ref({})
|
||||
|
||||
const defects = computed(() =>
|
||||
(overview.value?.items || []).filter((i) => i.triage_status === 'defect'))
|
||||
|
||||
const classSummary = computed(() => {
|
||||
const bc = overview.value?.by_class || {}
|
||||
return Object.entries(bc)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(([k, n]) => `${k.replaceAll('_', ' ')} ${n}`)
|
||||
.join(' · ')
|
||||
})
|
||||
|
||||
onMounted(refresh)
|
||||
|
||||
async function refresh() {
|
||||
loading.value = true
|
||||
try {
|
||||
overview.value = await store.errors()
|
||||
} catch (e) {
|
||||
toast({ text: `Could not load failed jobs: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onProbe() {
|
||||
probing.value = true
|
||||
try {
|
||||
await store.triageErrors()
|
||||
toast({ text: 'Probe queued — verdicts appear here as files are checked (large videos take a while)', type: 'success' })
|
||||
} catch (e) {
|
||||
toast({ text: `Could not start the probe: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
probing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onRecover(it) {
|
||||
recovering.value = it.image_id
|
||||
try {
|
||||
const res = await store.recoverImage(it.image_id)
|
||||
if (res.status === 'refetch_queued') {
|
||||
toast({ text: `Deleted the bad copy and queued a re-check of source #${res.source_id} — it re-imports on the next fetch`, type: 'success' })
|
||||
await refresh()
|
||||
} else if (res.status === 'no_source') {
|
||||
recovered.value = { ...recovered.value, [it.image_id]: 'no_source' }
|
||||
toast({ text: 'No enabled subscription source covers this file — replace it manually', type: 'warning' })
|
||||
} else {
|
||||
toast({ text: 'Image record no longer exists — refreshing', type: 'warning' })
|
||||
await refresh()
|
||||
}
|
||||
} catch (e) {
|
||||
toast({ text: `Recovery failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
recovering.value = null
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-section-h {
|
||||
font-size: 13px; font-weight: 700; letter-spacing: 0.03em;
|
||||
text-transform: uppercase; color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
.fc-queue { display: flex; gap: 24px; }
|
||||
.fc-q__n {
|
||||
font-size: 20px; font-weight: 700; line-height: 1.1;
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
}
|
||||
.fc-q__l {
|
||||
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-good { color: rgb(var(--v-theme-success)); }
|
||||
.fc-weak { color: rgb(var(--v-theme-error)); }
|
||||
.fc-defect {
|
||||
display: flex; align-items: center; gap: 12px;
|
||||
background: rgb(var(--v-theme-surface-light)); border-radius: 8px;
|
||||
padding: 6px 10px;
|
||||
}
|
||||
.fc-defect__thumb {
|
||||
flex: 0 0 44px; width: 44px; height: 44px; border-radius: 6px;
|
||||
overflow: hidden; display: flex; align-items: center; justify-content: center;
|
||||
background: rgba(0, 0, 0, 0.25);
|
||||
}
|
||||
.fc-defect__thumb img { width: 100%; height: 100%; object-fit: cover; }
|
||||
.fc-defect__meta { flex: 1; min-width: 0; }
|
||||
.fc-defect__err {
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
</style>
|
||||
@@ -13,6 +13,7 @@
|
||||
<ThumbnailBackfillCard />
|
||||
<ArchiveReextractCard />
|
||||
<MissingFileRepairCard />
|
||||
<GpuTriageCard />
|
||||
<DbMaintenanceCard />
|
||||
</div>
|
||||
</section>
|
||||
@@ -48,6 +49,7 @@ import MLBackfillCard from './MLBackfillCard.vue'
|
||||
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||
import MissingFileRepairCard from './MissingFileRepairCard.vue'
|
||||
import GpuTriageCard from './GpuTriageCard.vue'
|
||||
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
||||
import MLThresholdSliders from './MLThresholdSliders.vue'
|
||||
import HeadsCard from './HeadsCard.vue'
|
||||
|
||||
@@ -36,10 +36,32 @@ export const useGpuStore = defineStore('gpu', () => {
|
||||
}
|
||||
|
||||
// Requeue ONLY the errored jobs (all task types) — the scoped recovery after
|
||||
// an agent fix, without re-running the done library. Returns { requeued }.
|
||||
// an agent fix, without re-running the done library. Triage-confirmed
|
||||
// defects stay put (they recover via recoverImage instead).
|
||||
// Returns { requeued, pruned, defects_kept }.
|
||||
async function retryErrors() {
|
||||
return await api.post('/api/gpu/retry_errors')
|
||||
}
|
||||
|
||||
return { token, rotateToken, status, backfill, reprocess, retryErrors }
|
||||
// Failure-triage view (#125): errored jobs joined with integrity verdicts.
|
||||
// Returns { total, by_class, triage, items }.
|
||||
async function errors() {
|
||||
return await api.get('/api/gpu/errors')
|
||||
}
|
||||
|
||||
// Run the file-probe sweep now instead of waiting out the 15-min beat.
|
||||
async function triageErrors() {
|
||||
return await api.post('/api/gpu/errors/triage')
|
||||
}
|
||||
|
||||
// Recover a defective original: delete the bad copy + record, re-poll its
|
||||
// Source. Returns { status: 'refetch_queued'|'no_source'|'not_found', ... }.
|
||||
async function recoverImage(imageId) {
|
||||
return await api.post(`/api/gpu/errors/${imageId}/recover`)
|
||||
}
|
||||
|
||||
return {
|
||||
token, rotateToken, status, backfill, reprocess, retryErrors,
|
||||
errors, triageErrors, recoverImage,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -213,3 +213,50 @@ async def test_retry_errors_requeues_only_errored(client, db):
|
||||
|
||||
st = await (await client.get("/api/gpu/status")).get_json()
|
||||
assert st["pending"] == 1 and st["error"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_errors_keeps_triaged_defects(client, db):
|
||||
"""A probe-confirmed DEFECT is a bad FILE — requeueing it just burns agent
|
||||
time re-minting the tombstone, so /retry_errors leaves it for the recovery
|
||||
surface and reports it as defects_kept."""
|
||||
img1 = await _img(db, "4" * 64)
|
||||
img2 = await _img(db, "5" * 64)
|
||||
db.add(GpuJob(image_record_id=img1.id, task="ccip", status="error",
|
||||
attempts=3, error="moov atom not found",
|
||||
triage_status="defect"))
|
||||
db.add(GpuJob(image_record_id=img2.id, task="ccip", status="error",
|
||||
attempts=3, error="ffmpeg timed out after 1200s"))
|
||||
await db.commit()
|
||||
|
||||
body = await (await client.post("/api/gpu/retry_errors")).get_json()
|
||||
assert body["requeued"] == 1
|
||||
assert body["defects_kept"] == 1
|
||||
|
||||
rows = dict((await db.execute(
|
||||
select(GpuJob.image_record_id, GpuJob.status)
|
||||
)).all())
|
||||
assert rows[img1.id] == "error" # defect stays tombstoned
|
||||
assert rows[img2.id] == "pending" # operational failure requeued
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_errors_endpoint_reports_triage_view(client, db):
|
||||
img = await _img(db, "6" * 64)
|
||||
db.add(GpuJob(image_record_id=img.id, task="ccip", status="error",
|
||||
attempts=3,
|
||||
error="no frames sampled from video — moov atom not found"))
|
||||
await db.commit()
|
||||
|
||||
resp = await client.get("/api/gpu/errors")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["total"] == 1
|
||||
assert body["by_class"] == {"truncated_or_corrupt": 1}
|
||||
assert body["triage"]["unclassified"] == 1
|
||||
item = body["items"][0]
|
||||
assert item["image_id"] == img.id
|
||||
assert item["task"] == "ccip"
|
||||
assert item["reason_class"] == "truncated_or_corrupt"
|
||||
assert item["triage_status"] is None
|
||||
assert item["image_url"].startswith("/images/")
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Failure triage (#125): probe errored jobs' files, flag verdicts, recover.
|
||||
|
||||
The probe is the arbiter: reason strings only bucket the overview. A file that
|
||||
passes checksum+decode is 'file_ok' (operational failure); anything else is a
|
||||
'defect' — surfaced for recovery and excluded from /retry_errors.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
from PIL import Image as PILImage
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
GpuJob,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
Source,
|
||||
)
|
||||
from backend.app.services.ml.gpu_triage import (
|
||||
classify_reason,
|
||||
recover_defective_image,
|
||||
triage_errored_jobs,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def test_classify_reason_buckets():
|
||||
assert classify_reason(
|
||||
"no frames sampled from video — moov atom not found"
|
||||
) == "truncated_or_corrupt"
|
||||
assert classify_reason("ffmpeg timed out after 1200s") == "timeout"
|
||||
assert classify_reason(
|
||||
"gave up after repeated transient failures: HTTPConnectionPool read timed out"
|
||||
) == "transient"
|
||||
assert classify_reason(
|
||||
"poisoned: 10+ lease attempts without ever completing"
|
||||
) == "poisoned"
|
||||
assert classify_reason("cannot identify image file") == "decode"
|
||||
assert classify_reason("something novel") == "other"
|
||||
assert classify_reason(None) == "other"
|
||||
|
||||
|
||||
async def _errored_image(db, tmp_path, *, name, sha, content: bytes | None,
|
||||
error="no frames sampled from video — moov atom not found"):
|
||||
"""An ImageRecord (file written iff content is not None) + an errored job."""
|
||||
path = tmp_path / name
|
||||
if content is not None:
|
||||
path.write_bytes(content)
|
||||
img = ImageRecord(
|
||||
path=str(path), sha256=sha, size_bytes=1, mime="image/png",
|
||||
width=1, height=1, origin="imported_filesystem",
|
||||
integrity_status="unknown",
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
db.add(GpuJob(image_record_id=img.id, task="ccip", status="error",
|
||||
error=error, attempts=3))
|
||||
await db.flush()
|
||||
return img
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_triage_probes_and_splits_defect_vs_file_ok(db, tmp_path):
|
||||
# Healthy: a real PNG whose recorded sha matches its bytes → file_ok.
|
||||
ok_path = tmp_path / "fine.png"
|
||||
PILImage.new("RGB", (4, 4), (200, 30, 30)).save(ok_path)
|
||||
ok_sha = hashlib.sha256(ok_path.read_bytes()).hexdigest()
|
||||
ok = await _errored_image(db, tmp_path, name="fine.png", sha=ok_sha,
|
||||
content=None, error="ffmpeg timed out after 1200s")
|
||||
# Corrupt: bytes don't match the recorded sha → defect.
|
||||
bad = await _errored_image(db, tmp_path, name="bad.png", sha="0" * 64,
|
||||
content=b"not a real png")
|
||||
# Missing: no file on disk at all → defect (failed_verification).
|
||||
gone = await _errored_image(db, tmp_path, name="gone.png", sha="1" * 64,
|
||||
content=None)
|
||||
await db.commit()
|
||||
|
||||
summary = await db.run_sync(lambda s: triage_errored_jobs(s))
|
||||
assert summary["probed"] == 3
|
||||
assert summary["defect"] == 2
|
||||
assert summary["file_ok"] == 1
|
||||
assert summary["partial"] is False
|
||||
|
||||
# Column selects, not ORM refresh — the sweep wrote via Core DML.
|
||||
rows = dict((await db.execute(
|
||||
select(GpuJob.image_record_id, GpuJob.triage_status)
|
||||
.where(GpuJob.status == "error")
|
||||
)).all())
|
||||
assert rows[ok.id] == "file_ok"
|
||||
assert rows[bad.id] == "defect"
|
||||
assert rows[gone.id] == "defect"
|
||||
verdicts = dict((await db.execute(
|
||||
select(ImageRecord.id, ImageRecord.integrity_status)
|
||||
.where(ImageRecord.id.in_([ok.id, bad.id, gone.id]))
|
||||
)).all())
|
||||
assert verdicts[ok.id] == "ok"
|
||||
assert verdicts[bad.id] == "corrupt"
|
||||
assert verdicts[gone.id] == "failed_verification"
|
||||
|
||||
# Idempotent: everything already triaged → no re-probe.
|
||||
again = await db.run_sync(lambda s: triage_errored_jobs(s))
|
||||
assert again["probed"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_without_pollable_source_reports_no_source(db, tmp_path):
|
||||
img = await _errored_image(db, tmp_path, name="orphan.png", sha="2" * 64,
|
||||
content=b"x")
|
||||
await db.commit()
|
||||
res = await db.run_sync(
|
||||
lambda s: recover_defective_image(s, img.id, images_root=tmp_path)
|
||||
)
|
||||
assert res["status"] == "no_source"
|
||||
still_there = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.id == img.id)
|
||||
)).scalar_one_or_none()
|
||||
assert still_there == img.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_deletes_record_and_requeues_source(
|
||||
client, db, tmp_path, monkeypatch,
|
||||
):
|
||||
img = await _errored_image(db, tmp_path, name="fixme.png", sha="3" * 64,
|
||||
content=b"x")
|
||||
artist = Artist(name="Recov", slug="recov")
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
src = Source(artist_id=artist.id, platform="patreon",
|
||||
url="https://www.patreon.com/recov", enabled=True)
|
||||
db.add(src)
|
||||
await db.flush()
|
||||
post = Post(artist_id=artist.id, source_id=src.id, external_post_id="p1")
|
||||
db.add(post)
|
||||
await db.flush()
|
||||
db.add(ImageProvenance(image_record_id=img.id, post_id=post.id,
|
||||
source_id=src.id))
|
||||
img_id, src_id = img.id, src.id
|
||||
await db.commit()
|
||||
|
||||
queued = []
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.download.download_source.delay",
|
||||
lambda sid: queued.append(sid),
|
||||
)
|
||||
|
||||
resp = await client.post(f"/api/gpu/errors/{img_id}/recover")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["status"] == "refetch_queued"
|
||||
assert body["source_id"] == src_id
|
||||
assert queued == [src_id]
|
||||
|
||||
# Record gone — the error tombstones cascade away with it.
|
||||
remaining = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.id == img_id)
|
||||
)).scalar_one_or_none()
|
||||
assert remaining is None
|
||||
jobs_left = (await db.execute(
|
||||
select(GpuJob.id).where(GpuJob.image_record_id == img_id)
|
||||
)).scalars().all()
|
||||
assert jobs_left == []
|
||||
Reference in New Issue
Block a user