fix: phase 3 stops importing at a deadline and leaves the rest for the next chunk
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 22s
CI and images / extension-test (push) Successful in 22s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Failing after 2m34s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 22s
CI and images / extension-test (push) Successful in 22s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Failing after 2m34s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
A Discord recapture chunk on #banana-land was killed at the 1350s soft limit twice (events 92354 on 2026-09-28 and 92623 on 2026-09-29). The walk's budget (CHUNK_TOTAL_SECONDS, #4407) assumed 2.5s per import; measured on those chunks, a real import of the source's doodles took about 5s, so ~290 imports the walk had planned to fit took about twice the time reserved. - PHASE3_DEADLINE_SECONDS (1200s from task start): on native platforms phase 3 starts no import past it. The files left stay on disk with no record and are not marked seen; the next chunk resumes on the page that holds the first of them and imports them through the existing "on disk but never imported" path. A walk that reached its end becomes PARTIAL so the backfill is not marked complete while files wait. - DownloadResult.written_cursors records each written file's page cursor; mark_seen_after_import takes skip_paths. - PHASE3_IMPORT_SECONDS 2.5 -> 5.0, the slow end of what was measured, so the walk rarely leans on the deadline. Task #4493. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -570,6 +570,64 @@ async def test_backfill_chunk_progress_advances_cursor(
|
||||
assert remaining == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_phase_3_leaves_imports_past_its_deadline_for_the_next_chunk(
|
||||
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
|
||||
):
|
||||
"""A walk that reached its end handed phase 3 more imports than fit before
|
||||
the deadline. Phase 3 imports what fits, leaves the rest on disk and
|
||||
unmarked, keeps the backfill running, and resumes it on the page holding
|
||||
the first file it left."""
|
||||
import backend.app.services.download_service as ds
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:PAGE1"}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||
f2 = images_root / "alice" / "patreon" / "post" / "b.jpg"
|
||||
_make_jpg(f1, split="h")
|
||||
_make_jpg(f2, split="v")
|
||||
result = _make_fake_dl_result(
|
||||
success=True, written_paths=[str(f1), str(f2)], cursor="03:PAGE3",
|
||||
)
|
||||
result.written_cursors = ["03:PAGE1", "03:PAGE2"]
|
||||
left_unmarked = []
|
||||
result.mark_seen_after_import = lambda skip_paths=(): left_unmarked.append(
|
||||
set(skip_paths)
|
||||
)
|
||||
|
||||
# The clock is past the deadline once the first import is done.
|
||||
past = ds.PHASE3_DEADLINE_SECONDS + 1
|
||||
ticks = iter([0.0, 0.0])
|
||||
monkeypatch.setattr(ds, "time", SimpleNamespace(monotonic=lambda: next(ticks, past)))
|
||||
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, result)
|
||||
event_id = await svc.download_source(source.id)
|
||||
|
||||
ev = (await db.execute(
|
||||
select(DownloadEvent).where(DownloadEvent.id == event_id)
|
||||
)).scalar_one()
|
||||
assert ev.status == "ok"
|
||||
assert ev.files_count == 1
|
||||
assert "1 file(s) left for the next chunk" in ev.metadata_["stdout"]
|
||||
assert left_unmarked == [{str(f2)}]
|
||||
assert f2.exists()
|
||||
assert db_sync.execute(
|
||||
select(func.count(ImageRecord.id)).where(ImageRecord.path == str(f2))
|
||||
).scalar_one() == 0
|
||||
|
||||
remaining, co = (await db.execute(
|
||||
select(Source.backfill_runs_remaining, Source.config_overrides)
|
||||
.where(Source.id == source.id)
|
||||
)).one()
|
||||
assert co.get("_backfill_state") == "running"
|
||||
assert co.get("_backfill_cursor") == "03:PAGE2"
|
||||
assert remaining == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_state_running_selects_backfill_mode_and_resumes(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
|
||||
@@ -55,6 +55,15 @@ def test_the_native_chunk_leaves_room_to_tear_down_before_the_soft_limit():
|
||||
assert CHUNK_TOTAL_SECONDS <= DOWNLOAD_SOFT_TIME_LIMIT - 120
|
||||
|
||||
|
||||
def test_phase_3_stops_importing_with_room_to_finalize():
|
||||
"""Past its deadline phase 3 starts no import; what remains after it (the
|
||||
import in hand, post records, the event) has to fit before the soft limit."""
|
||||
from backend.app.services.ingest_core import PHASE3_DEADLINE_SECONDS
|
||||
from backend.app.tasks.download import DOWNLOAD_SOFT_TIME_LIMIT
|
||||
|
||||
assert PHASE3_DEADLINE_SECONDS <= DOWNLOAD_SOFT_TIME_LIMIT - 120
|
||||
|
||||
|
||||
def test_decorated_limits_match_module_constants():
|
||||
"""The @celery.task decorator must use the audited constants, not
|
||||
drifted literals."""
|
||||
|
||||
@@ -984,6 +984,31 @@ async def test_recapture_imports_a_seen_file_nothing_imported(
|
||||
assert downloader.download_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_phase_3_leaves_unimported_stay_unmarked(
|
||||
source_id, sync_engine, tmp_path,
|
||||
):
|
||||
"""Phase 3 can stop importing at its deadline. Each written file names the
|
||||
page it came from, so the next chunk can resume there; the files left keep
|
||||
no ledger mark, so that walk meets them on disk and imports them."""
|
||||
m1, m2 = _media("p1", 1), _media("p2", 1)
|
||||
pages = [("CUR1", [("p1", [m1])]), ("CUR2", [("p2", [m2])])]
|
||||
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
|
||||
result = ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="backfill")
|
||||
assert result.written_cursors == ["CUR1", "CUR2"]
|
||||
|
||||
result.mark_seen_after_import(skip_paths={str(tmp_path / "p2_1.jpg")})
|
||||
assert _count_ledger(sync_engine, source_id) == 3 # m1 and both post records
|
||||
|
||||
ing2 = _ingester(sync_engine, tmp_path, _FakeClient(pages),
|
||||
_FakeDownloader(tmp_path, on_disk={_ledger_key(m1), _ledger_key(m2)}))
|
||||
again = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="backfill")
|
||||
assert again.written_paths == [str(tmp_path / "p2_1.jpg")]
|
||||
assert "on disk but never imported: p2_1.jpg" in again.stdout
|
||||
|
||||
|
||||
# --- dead-letter ledger (plan #705 #7) ------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user