a7abcc41ca
An errored GPU job's stored reason is a suspicion; the file probe is the
verdict. A 15-min beat sweep (triage_gpu_errors) runs verify_integrity's own
probe (sha256 + decode) on each errored image ONCE and writes both verdicts:
ImageRecord.integrity_status and the new GpuJob.triage_status ('defect' |
'file_ok', migration 0072). Every classification logs at WARNING so it
surfaces in Logs/System Activity.
- 'defect' rows are excluded from /retry_errors (re-running a known-bad file
burns agent time re-minting the tombstone); response now reports
defects_kept and the GpuAgentCard toast says so.
- GET /api/gpu/errors: triage view — reason buckets (classify_reason),
probe verdicts, per-job detail. POST /errors/triage runs the sweep now.
- POST /api/gpu/errors/<id>/recover: reuses the Layer-2 refetch pattern —
delete the defective copy + record (full cascade takes the tombstones too)
and re-poll its subscription Source so a fresh copy re-imports and re-enters
the pipeline; 'no_source' when nothing pollable resolves.
- New 'Failed processing' card (GpuTriageCard) in Maintenance: verdict counts,
reason summary, probe-now, defect list with thumbnails + per-image Recover.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
190 lines
7.2 KiB
Vue
190 lines
7.2 KiB
Vue
<template>
|
|
<MaintenanceTile
|
|
icon="mdi-file-alert"
|
|
title="Failed processing"
|
|
blurb="Triage originals that failed GPU processing — probe the files, flag defects, recover them."
|
|
>
|
|
<p class="fc-muted text-body-2 mb-3">
|
|
A job that keeps failing parks as an error with its reason. A background
|
|
probe then checks the FILE itself (checksum + decode) and splits the
|
|
errors: <b>defective files</b> (truncated/corrupt originals — listed below
|
|
for recovery) vs <b>file OK</b> (the failure was operational; requeue
|
|
those with <i>Retry errored jobs</i> on the GPU agent card).
|
|
</p>
|
|
|
|
<div v-if="loading" class="fc-muted text-body-2">Loading…</div>
|
|
<template v-else-if="overview">
|
|
<div class="fc-queue mb-3">
|
|
<div class="fc-q"><div class="fc-q__n">{{ overview.total }}</div><div class="fc-q__l">errored</div></div>
|
|
<div class="fc-q"><div class="fc-q__n" :class="overview.triage.defect ? 'fc-weak' : ''">{{ overview.triage.defect }}</div><div class="fc-q__l">defective</div></div>
|
|
<div class="fc-q"><div class="fc-q__n fc-good">{{ overview.triage.file_ok }}</div><div class="fc-q__l">file ok</div></div>
|
|
<div class="fc-q"><div class="fc-q__n">{{ overview.triage.unclassified }}</div><div class="fc-q__l">unprobed</div></div>
|
|
</div>
|
|
|
|
<p v-if="classSummary" class="fc-muted text-caption mb-3">
|
|
Reasons: {{ classSummary }}
|
|
</p>
|
|
|
|
<div class="d-flex mb-4" style="gap:8px">
|
|
<v-btn
|
|
size="small" color="accent" variant="tonal" rounded="pill"
|
|
prepend-icon="mdi-magnify-scan" :loading="probing"
|
|
:disabled="!overview.triage.unclassified" @click="onProbe"
|
|
>Probe unclassified now</v-btn>
|
|
<v-btn
|
|
size="small" variant="text" rounded="pill"
|
|
prepend-icon="mdi-refresh" @click="refresh"
|
|
>Refresh</v-btn>
|
|
</div>
|
|
|
|
<template v-if="defects.length">
|
|
<div class="fc-section-h mb-2">Defective originals</div>
|
|
<div v-for="it in defects" :key="it.job_id" class="fc-defect mb-2">
|
|
<a :href="it.image_url" target="_blank" rel="noopener" class="fc-defect__thumb">
|
|
<img v-if="it.thumbnail_url" :src="it.thumbnail_url" alt="">
|
|
<v-icon v-else icon="mdi-file-question" size="28" />
|
|
</a>
|
|
<div class="fc-defect__meta">
|
|
<div class="text-body-2">
|
|
image <b>{{ it.image_id }}</b> · {{ it.task }} ·
|
|
<span class="fc-weak">{{ it.integrity_status }}</span>
|
|
</div>
|
|
<div class="fc-muted text-caption fc-defect__err" :title="it.error || ''">
|
|
{{ it.error || 'no stored reason' }}
|
|
</div>
|
|
</div>
|
|
<v-btn
|
|
v-if="recovered[it.image_id] !== 'no_source'"
|
|
size="small" color="accent" variant="tonal" rounded="pill"
|
|
prepend-icon="mdi-cloud-download" :loading="recovering === it.image_id"
|
|
@click="onRecover(it)"
|
|
>Recover</v-btn>
|
|
<span v-else class="fc-muted text-caption">
|
|
no pollable source — replace the file manually
|
|
</span>
|
|
</div>
|
|
<p class="fc-muted text-caption mt-2 mb-0">
|
|
Recover deletes the bad copy (and its record) and re-checks its
|
|
subscription source, so a fresh download re-imports it and re-enters
|
|
processing. Files without a pollable source need manual replacement.
|
|
</p>
|
|
</template>
|
|
<p v-else-if="!overview.total" class="fc-muted text-body-2 mb-0">
|
|
No failed jobs — the pipeline is clean.
|
|
</p>
|
|
<p v-else-if="!overview.triage.unclassified" class="fc-muted text-body-2 mb-0">
|
|
No defective files — every probed failure was operational
|
|
(file OK). Requeue them from the GPU agent card.
|
|
</p>
|
|
</template>
|
|
</MaintenanceTile>
|
|
</template>
|
|
|
|
<script setup>
|
|
import { computed, onMounted, ref } from 'vue'
|
|
|
|
import { toast } from '../../utils/toast.js'
|
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
|
import { useGpuStore } from '../../stores/gpu.js'
|
|
|
|
const store = useGpuStore()
|
|
const loading = ref(true)
|
|
const overview = ref(null)
|
|
const probing = ref(false)
|
|
const recovering = ref(null)
|
|
// image_id -> 'no_source' for rows recovery already declined; keeps the
|
|
// verdict visible instead of a button that fails the same way again.
|
|
const recovered = ref({})
|
|
|
|
const defects = computed(() =>
|
|
(overview.value?.items || []).filter((i) => i.triage_status === 'defect'))
|
|
|
|
const classSummary = computed(() => {
|
|
const bc = overview.value?.by_class || {}
|
|
return Object.entries(bc)
|
|
.sort((a, b) => b[1] - a[1])
|
|
.map(([k, n]) => `${k.replaceAll('_', ' ')} ${n}`)
|
|
.join(' · ')
|
|
})
|
|
|
|
onMounted(refresh)
|
|
|
|
async function refresh() {
|
|
loading.value = true
|
|
try {
|
|
overview.value = await store.errors()
|
|
} catch (e) {
|
|
toast({ text: `Could not load failed jobs: ${e.message}`, type: 'error' })
|
|
} finally {
|
|
loading.value = false
|
|
}
|
|
}
|
|
|
|
async function onProbe() {
|
|
probing.value = true
|
|
try {
|
|
await store.triageErrors()
|
|
toast({ text: 'Probe queued — verdicts appear here as files are checked (large videos take a while)', type: 'success' })
|
|
} catch (e) {
|
|
toast({ text: `Could not start the probe: ${e.message}`, type: 'error' })
|
|
} finally {
|
|
probing.value = false
|
|
}
|
|
}
|
|
|
|
async function onRecover(it) {
|
|
recovering.value = it.image_id
|
|
try {
|
|
const res = await store.recoverImage(it.image_id)
|
|
if (res.status === 'refetch_queued') {
|
|
toast({ text: `Deleted the bad copy and queued a re-check of source #${res.source_id} — it re-imports on the next fetch`, type: 'success' })
|
|
await refresh()
|
|
} else if (res.status === 'no_source') {
|
|
recovered.value = { ...recovered.value, [it.image_id]: 'no_source' }
|
|
toast({ text: 'No enabled subscription source covers this file — replace it manually', type: 'warning' })
|
|
} else {
|
|
toast({ text: 'Image record no longer exists — refreshing', type: 'warning' })
|
|
await refresh()
|
|
}
|
|
} catch (e) {
|
|
toast({ text: `Recovery failed: ${e.message}`, type: 'error' })
|
|
} finally {
|
|
recovering.value = null
|
|
}
|
|
}
|
|
</script>
|
|
|
|
<style scoped>
|
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
|
.fc-section-h {
|
|
font-size: 13px; font-weight: 700; letter-spacing: 0.03em;
|
|
text-transform: uppercase; color: rgb(var(--v-theme-on-surface));
|
|
}
|
|
.fc-queue { display: flex; gap: 24px; }
|
|
.fc-q__n {
|
|
font-size: 20px; font-weight: 700; line-height: 1.1;
|
|
font-family: 'JetBrains Mono', monospace;
|
|
}
|
|
.fc-q__l {
|
|
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
|
color: rgb(var(--v-theme-on-surface-variant));
|
|
}
|
|
.fc-good { color: rgb(var(--v-theme-success)); }
|
|
.fc-weak { color: rgb(var(--v-theme-error)); }
|
|
.fc-defect {
|
|
display: flex; align-items: center; gap: 12px;
|
|
background: rgb(var(--v-theme-surface-light)); border-radius: 8px;
|
|
padding: 6px 10px;
|
|
}
|
|
.fc-defect__thumb {
|
|
flex: 0 0 44px; width: 44px; height: 44px; border-radius: 6px;
|
|
overflow: hidden; display: flex; align-items: center; justify-content: center;
|
|
background: rgba(0, 0, 0, 0.25);
|
|
}
|
|
.fc-defect__thumb img { width: 100%; height: 100%; object-fit: cover; }
|
|
.fc-defect__meta { flex: 1; min-width: 0; }
|
|
.fc-defect__err {
|
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
|
}
|
|
</style>
|