fix(maintenance): time-box + self-resume the archive re-extract task
reextract_archive_attachments loaded ALL PostAttachments and ran in one pass up to a 30-min soft limit, then died without re-enqueueing — a large archive backlog would only ever partially process. And a naive re-run can't advance: an already-extracted archive is still an archive on disk, so it'd re-extract the same first batch forever. Give it a real cursor + time-box + self-resume (mirrors normalize_tags_task, operator-asked 2026-06-07: reasonable timeout, then re-queue so other work keeps flowing): - service scans attachments with id > after_id in ascending order, time-boxes the chunk, and reports partial=True + resume_after_id (last scanned id). - task passes a 600s budget and re-enqueues itself from the cursor until the scan is exhausted. Routes on the maintenance_long lane. - This is independent of the maintenance_long lane isolation (already shipped) — that stops long tasks starving the quick maintenance queue; this stops the re-extract itself dying on a big backlog. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -89,3 +89,67 @@ def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch)
|
||||
)
|
||||
assert again["members_imported"] == 0
|
||||
assert db_sync.execute(select(ImageRecord)).scalars().all() == images
|
||||
|
||||
|
||||
def test_reextract_timebox_resumes_from_cursor(db_sync, tmp_path, monkeypatch):
|
||||
"""A 0-second budget cuts the chunk after the first attachment and reports a
|
||||
resume cursor; the next run starts strictly after it and finishes the rest."""
|
||||
from backend.app.tasks import ml as ml_mod
|
||||
from backend.app.tasks import thumbnail as thumb_mod
|
||||
|
||||
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
|
||||
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
images_root.mkdir()
|
||||
|
||||
artist = Artist(name="Bob", slug="bob")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
source = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/bob", enabled=True, config_overrides={},
|
||||
)
|
||||
db_sync.add(source)
|
||||
db_sync.flush()
|
||||
post = Post(
|
||||
source_id=source.id, artist_id=artist.id, external_post_id="42",
|
||||
post_url="https://www.patreon.com/posts/42",
|
||||
)
|
||||
db_sync.add(post)
|
||||
db_sync.flush()
|
||||
|
||||
store_dir = images_root / "attachments" / "two"
|
||||
store_dir.mkdir(parents=True)
|
||||
att_ids = []
|
||||
for n, color in enumerate(("red", "blue")):
|
||||
arc = store_dir / f"{n}_archive_{color}"
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr(f"{color}.jpg", _jpeg(color))
|
||||
sha = hashlib.sha256(arc.read_bytes()).hexdigest()
|
||||
att = PostAttachment(
|
||||
post_id=post.id, artist_id=artist.id, sha256=sha, path=str(arc),
|
||||
original_filename=arc.name, ext="", size_bytes=arc.stat().st_size,
|
||||
)
|
||||
db_sync.add(att)
|
||||
db_sync.flush()
|
||||
att_ids.append(att.id)
|
||||
db_sync.commit()
|
||||
|
||||
# Budget 0 → break right after the first attachment commits.
|
||||
first = cleanup_service.reextract_archive_attachments(
|
||||
db_sync, images_root=images_root, time_budget_seconds=0.0,
|
||||
)
|
||||
assert first["partial"] is True
|
||||
assert first["scanned"] == 1
|
||||
assert first["members_imported"] == 1
|
||||
assert first["resume_after_id"] == att_ids[0]
|
||||
|
||||
# Resume strictly after the cursor — picks up the second, then runs dry.
|
||||
second = cleanup_service.reextract_archive_attachments(
|
||||
db_sync, images_root=images_root, after_id=att_ids[0],
|
||||
)
|
||||
assert second["partial"] is False
|
||||
assert second["scanned"] == 1
|
||||
assert second["members_imported"] == 1
|
||||
assert len(db_sync.execute(select(ImageRecord)).scalars().all()) == 2
|
||||
|
||||
Reference in New Issue
Block a user