feat(download): smarter backfill — time-boxed chunks, run-until-done (backend)
Plan #693. Large-catalog backfill (Anduo) no longer sprints to the timeout wall and dies as an error each run. Builds on the cursor checkpoint (#689). - Time-boxed chunks: BACKFILL_TIMEOUT_SECONDS(1170)→BACKFILL_CHUNK_SECONDS(600), far under the 1350 soft limit. Hitting it = normal chunk boundary (the TimeoutExpired path already captures partial output + the cursor), not a near-wall death. - Run-until-done state machine driven by config_overrides[_backfill_state] (running/complete/stalled). A running backfill auto-continues in chunks across ticks until gallery-dl exits cleanly (rc=0 = reached the bottom → 'complete'); a safety-cap (BACKFILL_MAX_CHUNKS=200) + the #689 stall-guard pause a pathological walk as 'stalled'. Replaces the N-runs counter (backfill_runs_remaining repurposed as the cap countdown). - Progress, not error: a chunk that timed out but advanced (cursor moved and/or files written) is reclassified TIMEOUT→PARTIAL (status 'ok'). - Retry storm tamed: gallery-dl retries 3→2, downloader timeout 120→60s, so one stuck CDN file fails in ~1-2 min not ~10 (Anduo #40838). - API: POST /sources/{id}/backfill now takes {action: start|stop}; service start_backfill/stop_backfill; new enabled sources auto-arm run-until-done; source dict exposes backfill_state + backfill_chunks. Frontend (Start/Stop control + state badge) lands in the next push. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+13
-14
@@ -199,7 +199,7 @@ async def test_list_derives_next_check_at_when_last_checked_set(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_arms_source(client, artist, db):
|
||||
async def test_backfill_endpoint_start_and_stop(client, artist, db):
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-backfill", enabled=True,
|
||||
@@ -208,18 +208,21 @@ async def test_backfill_endpoint_arms_source(client, artist, db):
|
||||
await db.commit()
|
||||
sid = src.id
|
||||
|
||||
resp = await client.post(f"/api/sources/{sid}/backfill", json={"runs": 5})
|
||||
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "start"})
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["backfill_runs_remaining"] == 5
|
||||
assert body["backfill_state"] == "running"
|
||||
|
||||
# GET reflects the new state.
|
||||
# GET reflects the running state.
|
||||
one = await client.get(f"/api/sources/{sid}")
|
||||
assert (await one.get_json())["backfill_runs_remaining"] == 5
|
||||
assert (await one.get_json())["backfill_state"] == "running"
|
||||
|
||||
stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
|
||||
assert (await stopped.get_json())["backfill_state"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_defaults_to_three(client, artist, db):
|
||||
async def test_backfill_endpoint_defaults_to_start(client, artist, db):
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-backfill-default", enabled=True,
|
||||
@@ -228,26 +231,22 @@ async def test_backfill_endpoint_defaults_to_three(client, artist, db):
|
||||
await db.commit()
|
||||
resp = await client.post(f"/api/sources/{src.id}/backfill", json={})
|
||||
body = await resp.get_json()
|
||||
assert body["backfill_runs_remaining"] == 3
|
||||
assert body["backfill_state"] == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_rejects_out_of_range(client, artist, db):
|
||||
async def test_backfill_endpoint_rejects_bad_action(client, artist, db):
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-backfill-bad", enabled=True,
|
||||
)
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
bad = await client.post(f"/api/sources/{src.id}/backfill", json={"runs": 0})
|
||||
bad = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "nope"})
|
||||
assert bad.status_code == 400
|
||||
too_big = await client.post(
|
||||
f"/api/sources/{src.id}/backfill", json={"runs": 99}
|
||||
)
|
||||
assert too_big.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_404_when_source_missing(client):
|
||||
resp = await client.post("/api/sources/999999/backfill", json={"runs": 3})
|
||||
resp = await client.post("/api/sources/999999/backfill", json={"action": "start"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
Reference in New Issue
Block a user