feat(download): smarter backfill — time-boxed chunks, run-until-done (backend)
Plan #693. Large-catalog backfill (Anduo) no longer sprints to the timeout wall and dies as an error each run. Builds on the cursor checkpoint (#689). - Time-boxed chunks: BACKFILL_TIMEOUT_SECONDS(1170)→BACKFILL_CHUNK_SECONDS(600), far under the 1350 soft limit. Hitting it = normal chunk boundary (the TimeoutExpired path already captures partial output + the cursor), not a near-wall death. - Run-until-done state machine driven by config_overrides[_backfill_state] (running/complete/stalled). A running backfill auto-continues in chunks across ticks until gallery-dl exits cleanly (rc=0 = reached the bottom → 'complete'); a safety-cap (BACKFILL_MAX_CHUNKS=200) + the #689 stall-guard pause a pathological walk as 'stalled'. Replaces the N-runs counter (backfill_runs_remaining repurposed as the cap countdown). - Progress, not error: a chunk that timed out but advanced (cursor moved and/or files written) is reclassified TIMEOUT→PARTIAL (status 'ok'). - Retry storm tamed: gallery-dl retries 3→2, downloader timeout 120→60s, so one stuck CDN file fails in ~1-2 min not ~10 (Anduo #40838). - API: POST /sources/{id}/backfill now takes {action: start|stop}; service start_backfill/stop_backfill; new enabled sources auto-arm run-until-done; source dict exposes backfill_state + backfill_chunks. Frontend (Start/Stop control + state badge) lands in the next push. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+127
-155
@@ -372,30 +372,16 @@ async def test_finalize_skipped_preserves_failures_clears_error(db):
|
||||
assert row.last_checked_at is not None
|
||||
|
||||
|
||||
# --- Plan #544: backfill lifecycle + PARTIAL → status=ok -------------------
|
||||
# --- Plan #693: backfill state machine (time-boxed chunks, run-until-done) ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_decrements_and_checkpoints_cursor(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""A backfill run that did NOT finish the walk (subprocess budget cut it
|
||||
short → success=False / rc!=0) decrements the run budget AND checkpoints
|
||||
gallery-dl's last-emitted cursor, so the next run resumes instead of
|
||||
re-walking from the top (plan #689, Anduo #40411)."""
|
||||
def _backfill_svc(db, db_sync, tmp_path, result):
|
||||
"""DownloadService wired with a fake gdl returning `result` + a real
|
||||
importer (so an empty written_paths just attaches nothing)."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.backfill_runs_remaining = 3
|
||||
await db.commit()
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||
success=False, written_paths=[], files_downloaded=0,
|
||||
stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n",
|
||||
))
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
@@ -404,122 +390,138 @@ async def test_backfill_decrements_and_checkpoints_cursor(
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
fake_gdl = _fake_gdl_with_result(result)
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining, overrides = (await db.execute(
|
||||
select(Source.backfill_runs_remaining, Source.config_overrides)
|
||||
.where(Source.id == source.id)
|
||||
)).one()
|
||||
assert remaining == 2
|
||||
assert overrides.get("_backfill_cursor") == "03:PAGE2:xyz"
|
||||
return svc, fake_gdl
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_cursor_keeps_backfill_mode_after_budget_drained(
|
||||
async def test_backfill_chunk_progress_advances_cursor(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""A pending cursor with runs_remaining == 0 still runs in BACKFILL mode
|
||||
(skip:True + the resume cursor threaded to gallery-dl), so a large walk
|
||||
finishes across ticks without the operator re-triggering."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
"""A running backfill chunk that didn't finish but advanced (new cursor)
|
||||
stays 'running', checkpoints the cursor, bumps the chunk counter, and
|
||||
spends one safety-cap chunk."""
|
||||
_artist, source = seed_artist_and_source
|
||||
source.backfill_runs_remaining = 0
|
||||
source.config_overrides = {"_backfill_cursor": "03:RESUME:here"}
|
||||
source.config_overrides = {"_backfill_state": "running"}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=False, written_paths=[], files_downloaded=0,
|
||||
stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n",
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining, co = (await db.execute(
|
||||
select(Source.backfill_runs_remaining, Source.config_overrides)
|
||||
.where(Source.id == source.id)
|
||||
)).one()
|
||||
assert co.get("_backfill_state") == "running"
|
||||
assert co.get("_backfill_cursor") == "03:PAGE2:xyz"
|
||||
assert co.get("_backfill_chunks") == 1
|
||||
assert remaining == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_state_running_selects_backfill_mode_and_resumes(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""state=='running' drives backfill mode (skip:True + chunk budget) and
|
||||
threads the stored cursor to gallery-dl as the resume point."""
|
||||
from backend.app.services.gallery_dl import (
|
||||
BACKFILL_CHUNK_SECONDS,
|
||||
BACKFILL_SKIP_VALUE,
|
||||
)
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:RESUME:here"}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
svc, fake_gdl = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=False, written_paths=[],
|
||||
stderr="[patreon][debug] Cursor: 03:RESUME2:next\n",
|
||||
))
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=tmp_path / "images",
|
||||
import_root=tmp_path / "images",
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
kwargs = fake_gdl.download.call_args.kwargs
|
||||
assert kwargs["skip_value"] is BACKFILL_SKIP_VALUE
|
||||
assert kwargs["source_config"].resume_cursor == "03:RESUME:here"
|
||||
assert kwargs["source_config"].timeout == BACKFILL_CHUNK_SECONDS
|
||||
|
||||
overrides = (await db.execute(
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert overrides.get("_backfill_cursor") == "03:RESUME2:next"
|
||||
assert co.get("_backfill_cursor") == "03:RESUME2:next"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_completes_and_clears_cursor(
|
||||
async def test_backfill_clean_exit_marks_complete(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""A clean rc=0 finish means gallery-dl walked to the bottom — clear the
|
||||
checkpoint and drop back to tick mode, regardless of files this run."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
"""A clean rc=0 chunk = gallery-dl reached the bottom → state 'complete',
|
||||
cursor cleared, returns to tick mode."""
|
||||
_artist, source = seed_artist_and_source
|
||||
source.backfill_runs_remaining = 2
|
||||
source.config_overrides = {"_backfill_cursor": "03:NEAR:bottom"}
|
||||
source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:NEAR:bottom"}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=True, written_paths=[], files_downloaded=0,
|
||||
stderr="[patreon][debug] Cursor: 03:LAST:page\n",
|
||||
))
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=tmp_path / "images",
|
||||
import_root=tmp_path / "images",
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining, overrides = (await db.execute(
|
||||
remaining, co = (await db.execute(
|
||||
select(Source.backfill_runs_remaining, Source.config_overrides)
|
||||
.where(Source.id == source.id)
|
||||
)).one()
|
||||
assert co.get("_backfill_state") == "complete"
|
||||
assert "_backfill_cursor" not in co
|
||||
assert remaining == 0
|
||||
assert "_backfill_cursor" not in (overrides or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_cursor_stuck_guard_gives_up(
|
||||
async def test_backfill_cap_exhaustion_stalls(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""If two consecutive non-completing runs fail to advance the cursor,
|
||||
drop the checkpoint + zero the budget so a wedged walk can't re-strand
|
||||
forever (cf. download soft-limit ladder)."""
|
||||
"""A progressing chunk that spends the last safety-cap chunk without
|
||||
reaching the bottom pauses as 'stalled' (doesn't loop forever)."""
|
||||
_artist, source = seed_artist_and_source
|
||||
source.config_overrides = {"_backfill_state": "running"}
|
||||
source.backfill_runs_remaining = 1
|
||||
await db.commit()
|
||||
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=False, written_paths=[],
|
||||
stderr="[patreon][debug] Cursor: 03:MORE:left\n",
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining, co = (await db.execute(
|
||||
select(Source.backfill_runs_remaining, Source.config_overrides)
|
||||
.where(Source.id == source.id)
|
||||
)).one()
|
||||
assert co.get("_backfill_state") == "stalled"
|
||||
assert remaining == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_stuck_guard_stalls_after_two_no_advance(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""Two consecutive chunks that fail to advance the cursor → 'stalled',
|
||||
cursor cleared, so a wedged walk can't re-strand forever."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from backend.app.services.download_service import DownloadService
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.backfill_runs_remaining = 0
|
||||
source.config_overrides = {"_backfill_cursor": "stuck"}
|
||||
source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "stuck"}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
svc = DownloadService(
|
||||
@@ -528,109 +530,79 @@ async def test_backfill_cursor_stuck_guard_gives_up(
|
||||
)
|
||||
ctx = {
|
||||
"source_id": source.id, "platform": "patreon",
|
||||
"config_overrides": {"_backfill_cursor": "stuck"},
|
||||
"backfill_runs_remaining": 0,
|
||||
"config_overrides": {"_backfill_state": "running", "_backfill_cursor": "stuck"},
|
||||
"backfill_runs_remaining": 5,
|
||||
}
|
||||
stuck = _make_fake_dl_result(success=False, stderr="Cursor: stuck\n")
|
||||
|
||||
# Run 1: cursor unchanged → 1 stall, checkpoint retained.
|
||||
await svc._apply_backfill_lifecycle(ctx, stuck)
|
||||
await db.commit()
|
||||
overrides = (await db.execute(
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert overrides.get("_backfill_cursor") == "stuck"
|
||||
assert overrides.get("_backfill_cursor_stalls") == 1
|
||||
assert co.get("_backfill_state") == "running"
|
||||
assert co.get("_backfill_cursor_stalls") == 1
|
||||
|
||||
# Run 2: still stuck → give up.
|
||||
ctx["config_overrides"] = dict(overrides)
|
||||
ctx["config_overrides"] = dict(co)
|
||||
await svc._apply_backfill_lifecycle(ctx, stuck)
|
||||
await db.commit()
|
||||
remaining, overrides2 = (await db.execute(
|
||||
select(Source.backfill_runs_remaining, Source.config_overrides)
|
||||
.where(Source.id == source.id)
|
||||
)).one()
|
||||
assert remaining == 0
|
||||
assert "_backfill_cursor" not in (overrides2 or {})
|
||||
co2 = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert co2.get("_backfill_state") == "stalled"
|
||||
assert "_backfill_cursor" not in co2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_auto_resets_on_clean_zero_files(
|
||||
async def test_backfill_timeout_chunk_reclassified_to_ok(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""A clean run (rc=0) that downloaded zero files means the backfill
|
||||
queue drained — reset to 0 immediately instead of wasting the rest of
|
||||
the N-run budget on no-op walks."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
"""A backfill chunk that hit its time-box (TIMEOUT) but advanced is
|
||||
reclassified to PARTIAL → event status 'ok' (progress, not error), and
|
||||
consecutive_failures is not bumped."""
|
||||
from backend.app.services.gallery_dl import ErrorType
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.backfill_runs_remaining = 3
|
||||
source.config_overrides = {"_backfill_state": "running"}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
fake_result = _make_fake_dl_result(
|
||||
success=True, written_paths=[], files_downloaded=0,
|
||||
)
|
||||
fake_gdl = _fake_gdl_with_result(fake_result)
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=False, files_downloaded=3,
|
||||
error_type=ErrorType.TIMEOUT,
|
||||
error_message="Download timed out",
|
||||
stderr="[patreon][debug] Cursor: 03:NEXT:page\n",
|
||||
))
|
||||
event_id = await svc.download_source(source.id)
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=tmp_path / "images",
|
||||
import_root=tmp_path / "images",
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining = (await db.execute(
|
||||
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||
ev = (await db.execute(
|
||||
select(DownloadEvent).where(DownloadEvent.id == event_id)
|
||||
)).scalar_one()
|
||||
assert remaining == 0
|
||||
assert ev.status == "ok"
|
||||
failures = (await db.execute(
|
||||
select(Source.consecutive_failures).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert failures == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_mode_does_not_touch_backfill_counter(
|
||||
async def test_tick_mode_when_not_running_leaves_state_untouched(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""When backfill_runs_remaining is already 0, downloads don't go
|
||||
negative or otherwise mutate the counter."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
"""No _backfill_state → not backfilling; the lifecycle is a no-op and the
|
||||
counter/state aren't mutated."""
|
||||
_artist, source = seed_artist_and_source
|
||||
assert source.backfill_runs_remaining == 0
|
||||
assert (source.config_overrides or {}).get("_backfill_state") is None
|
||||
|
||||
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=True, written_paths=[], files_downloaded=0,
|
||||
))
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=tmp_path / "images",
|
||||
import_root=tmp_path / "images",
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining = (await db.execute(
|
||||
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert remaining == 0
|
||||
assert (co or {}).get("_backfill_state") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user