fix(download): release DB connections across the gallery-dl subprocess
Backfill events were STILL stranding empty after the timeout-ladder fix.
Worker logs showed the salvage path working ("Download timeout for
anduo/patreon after 1170.0s (18 files written)") but then:
Retry in 3s: DBAPIError(ConnectionDoesNotExistError: connection was
closed in the middle of operation)
...succeeded in 0.149s <- in-flight guard no-op
Root cause: DownloadService held the async + sync DB connections checked
out across the entire (≤19.5-min backfill) gallery-dl subprocess. The
server reaps the idle connection, so phase 3's first query hits a dead
socket. That DBAPIError trips download_source's autoretry_for, the retry
re-enters _phase1_setup, sees the event still 'running', returns
in_flight and no-ops — leaving the event to be stranded empty by the
recovery sweep. pool_pre_ping was already on both engines but can't help
a *held* connection (it only validates on pool checkout).
Fix:
- DownloadService.download_source closes the async + sync sessions after
phase 1, before the subprocess, so phase 3 re-acquires a live
connection (matches the class's "Phase 2 — no DB connection" docstring).
- The per-task async engine switches to NullPool so phase 3 always opens
a fresh connection rather than a pooled one the server may have reaped.
Tests: assert connections are released before gdl.download runs and the
event still finalizes; assert the task engine uses NullPool. Also fixes a
stale 1800s->1170s comment.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -613,3 +613,81 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
|
||||
assert len(thumb_calls) == 2
|
||||
assert len(ml_calls) == 2
|
||||
assert sorted(thumb_calls) == sorted(ml_calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_releases_db_connections_before_subprocess(
|
||||
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
|
||||
):
|
||||
"""Phase 1's DB connections must be released BEFORE the (up to ~19.5-min
|
||||
in backfill) gallery-dl subprocess, so they don't idle-die and strand
|
||||
phase 3 with ConnectionDoesNotExistError (Anduo #40014). Spy on the
|
||||
async session close and assert it happened before gdl.download runs;
|
||||
phase 3 must still finalize the event."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
|
||||
closed = {"async": False}
|
||||
orig_close = db.close
|
||||
|
||||
async def spy_close():
|
||||
closed["async"] = True
|
||||
await orig_close()
|
||||
|
||||
monkeypatch.setattr(db, "close", spy_close)
|
||||
|
||||
seen = {}
|
||||
|
||||
async def fake_download(url, **k):
|
||||
seen["closed_before_download"] = closed["async"]
|
||||
return _make_fake_dl_result(success=True, written_paths=[], stdout="")
|
||||
|
||||
fake_gdl = MagicMock()
|
||||
fake_gdl.download = fake_download
|
||||
fake_gdl._compute_run_stats = lambda *a, **k: {
|
||||
"exit_code": 0, "downloaded_count": 0, "skipped_count": 0,
|
||||
"per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0,
|
||||
}
|
||||
fake_gdl._extract_errors_warnings = lambda *a, **k: ""
|
||||
fake_gdl._truncate_log = lambda x, **k: x
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=tmp_path, import_root=tmp_path,
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(
|
||||
db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
|
||||
)
|
||||
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
event_id = await svc.download_source(source.id)
|
||||
|
||||
assert seen["closed_before_download"] is True
|
||||
ev = (await db.execute(
|
||||
select(DownloadEvent).where(DownloadEvent.id == event_id)
|
||||
)).scalar_one()
|
||||
assert ev.status == "ok"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_task_engine_uses_nullpool():
|
||||
"""The per-task async engine must use NullPool so phase 3 re-acquires a
|
||||
fresh connection instead of a pooled one the server reaped during the
|
||||
long subprocess (Anduo #40014)."""
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from backend.app.tasks._async_session import async_session_factory
|
||||
|
||||
_factory, engine = async_session_factory()
|
||||
try:
|
||||
assert isinstance(engine.sync_engine.pool, NullPool)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
Reference in New Issue
Block a user