Smarter backfill: time-boxed chunks, run-until-done (plan #693) #71

Merged
bvandeusen merged 2 commits from dev into main 2026-06-05 16:33:07 -04:00
4 changed files with 55 additions and 39 deletions
Showing only changes of commit 618dafde85 - Show all commits
@@ -25,9 +25,17 @@
size="x-small" color="error" variant="tonal" label
>{{ source.consecutive_failures }} err</v-chip>
<v-chip
v-else-if="(source.backfill_runs_remaining || 0) > 0"
v-else-if="source.backfill_state === 'running'"
size="x-small" color="info" variant="tonal" label
>backfill ({{ source.backfill_runs_remaining }}×)</v-chip>
>Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }}</v-chip>
<v-chip
v-else-if="source.backfill_state === 'complete'"
size="x-small" color="success" variant="tonal" label
>Backfilled</v-chip>
<v-chip
v-else-if="source.backfill_state === 'stalled'"
size="x-small" color="warning" variant="tonal" label
>Stalled</v-chip>
</div>
<div class="fc-source-card__actions">
@@ -40,11 +48,13 @@
</v-btn>
<v-btn
size="x-small" variant="text"
:disabled="(source.backfill_runs_remaining || 0) > 0"
:color="source.backfill_state === 'running' ? 'warning' : undefined"
@click.stop="$emit('backfill', source)"
>
<v-icon>mdi-magnify-scan</v-icon>
<v-tooltip activator="parent" location="top">Deep scan</v-tooltip>
<v-icon>{{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
<v-tooltip activator="parent" location="top">
{{ source.backfill_state === 'running' ? 'Stop backfill' : 'Backfill full history' }}
</v-tooltip>
</v-btn>
<v-btn size="x-small" variant="text" @click.stop="$emit('edit', source)">
<v-icon>mdi-pencil</v-icon>
@@ -32,11 +32,17 @@
size="x-small" color="error" variant="tonal" label
>{{ source.consecutive_failures }}</v-chip>
<v-chip
v-else-if="(source.backfill_runs_remaining || 0) > 0"
v-else-if="source.backfill_state === 'running'"
size="x-small" color="info" variant="tonal" label
>
backfill ({{ source.backfill_runs_remaining }}×)
</v-chip>
>Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }}</v-chip>
<v-chip
v-else-if="source.backfill_state === 'complete'"
size="x-small" color="success" variant="tonal" label
>Backfilled</v-chip>
<v-chip
v-else-if="source.backfill_state === 'stalled'"
size="x-small" color="warning" variant="tonal" label
>Stalled</v-chip>
<span v-else class="fc-source-row__zero">0</span>
</td>
<td class="fc-source-row__actions">
@@ -49,13 +55,15 @@
<v-tooltip activator="parent" location="top">Check now</v-tooltip>
</v-btn>
<v-btn
icon="mdi-magnify-scan" size="x-small" variant="text"
:disabled="(source.backfill_runs_remaining || 0) > 0"
size="x-small" variant="text"
:color="source.backfill_state === 'running' ? 'warning' : undefined"
@click.stop="$emit('backfill', source)"
>
<v-icon>mdi-magnify-scan</v-icon>
<v-icon>{{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
<v-tooltip activator="parent" location="top">
Deep scan walk full history for next few runs
{{ source.backfill_state === 'running'
? 'Stop backfill'
: 'Backfill — walk the full post history until complete' }}
</v-tooltip>
</v-btn>
<v-btn
@@ -532,31 +532,23 @@ async function onCheck(source) {
}
}
// Plan #544: arm a source for backfill mode (gallery-dl walks the full
// post history) for the next N download runs. Default 3 — enough budget
// to finish a deep creator without re-prompting the operator across
// timeout boundaries. The chip on the row reflects the remaining count.
// Plan #693: toggle a run-until-done backfill. Starting walks the full post
// history in time-boxed chunks across ticks until it reaches the bottom (the
// row badge tracks progress / completion); stopping cancels back to tick mode.
async function onBackfill(source) {
const raw = globalThis.window?.prompt(
`Deep scan "${source.artist_name} (${source.platform})" — walk full history for the next how many download runs? (110, default 3)`,
'3',
)
if (raw == null) return
const runs = parseInt(raw, 10)
if (!Number.isFinite(runs) || runs < 1 || runs > 10) {
toast({ text: 'Deep scan: runs must be 110', type: 'error' })
return
}
const running = source.backfill_state === 'running'
try {
await store.setBackfill(source.id, runs, source.artist_id)
toast({
text: `Deep scan armed for ${runs} run${runs === 1 ? '' : 's'}`,
type: 'success',
})
if (running) {
await store.stopBackfill(source.id, source.artist_id)
toast({ text: `Backfill stopped for ${source.artist_name}`, type: 'success' })
} else {
await store.startBackfill(source.id, source.artist_id)
toast({ text: `Backfill started for ${source.artist_name}`, type: 'success' })
}
await store.loadAll()
} catch (e) {
toast({
text: `Deep scan failed: ${e?.detail || e?.message || e}`,
text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.detail || e?.message || e}`,
type: 'error',
})
}
+12 -6
View File
@@ -85,11 +85,16 @@ export const useSourcesStore = defineStore('sources', () => {
}
}
// Plan #544: arm a source for backfill mode. The next `runs` download
// runs (default 3) walk gallery-dl's full post history instead of
// exiting early at the first contiguous archived block.
async function setBackfill(id, runs = 3, artistIdHint = null) {
const body = await api.post(`/api/sources/${id}/backfill`, { body: { runs } })
// Plan #693: start/stop a run-until-done backfill. 'start' walks the full
// post history in time-boxed chunks until it reaches the bottom (then the
// source shows backfill_state 'complete'); 'stop' cancels back to tick mode.
async function startBackfill(id, artistIdHint = null) {
const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'start' } })
_invalidate(artistIdHint ?? body.artist_id)
return body
}
async function stopBackfill(id, artistIdHint = null) {
const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'stop' } })
_invalidate(artistIdHint ?? body.artist_id)
return body
}
@@ -120,7 +125,8 @@ export const useSourcesStore = defineStore('sources', () => {
loadAll, loadForArtist,
create, update, remove,
checkNow,
setBackfill,
startBackfill,
stopBackfill,
findOrCreateArtist, autocompleteArtist,
loadScheduleStatus,
sourcesByArtistGrouped,