fix(downloads): enqueue thumbnail + ML per attached image #46
@@ -260,6 +260,25 @@ class DownloadService:
|
|||||||
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
|
# Enqueue thumbnail + ML for newly-attached images, matching
|
||||||
|
# the filesystem-import path (tasks/import_file.py:228-239).
|
||||||
|
# Importer.attach_in_place deliberately skips inline thumb
|
||||||
|
# generation to keep the import queue moving; the calling
|
||||||
|
# task is responsible for the enqueue. Operator-flagged
|
||||||
|
# 2026-06-01: without this, every downloaded image stayed
|
||||||
|
# at thumbnail_path=NULL until a periodic backfill swept
|
||||||
|
# it up, surfacing as broken-thumbnail tiles in the gallery
|
||||||
|
# for hours after a download landed. Lazy import to avoid
|
||||||
|
# circular-import risk between this service and the
|
||||||
|
# tasks/* modules that import it.
|
||||||
|
from ..tasks.ml import tag_and_embed
|
||||||
|
from ..tasks.thumbnail import generate_thumbnail
|
||||||
|
ids = list(result.member_image_ids)
|
||||||
|
if result.image_id is not None and result.image_id not in ids:
|
||||||
|
ids.append(result.image_id)
|
||||||
|
for img_id in ids:
|
||||||
|
generate_thumbnail.delay(img_id)
|
||||||
|
tag_and_embed.delay(img_id)
|
||||||
elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
|
elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
|
||||||
"duplicate_hash", "duplicate_phash",
|
"duplicate_hash", "duplicate_phash",
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -546,3 +546,70 @@ async def test_partial_error_type_maps_to_ok_status(
|
|||||||
select(Source.consecutive_failures).where(Source.id == source.id)
|
select(Source.consecutive_failures).where(Source.id == source.id)
|
||||||
)).scalar_one()
|
)).scalar_one()
|
||||||
assert src_after == 0
|
assert src_after == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
|
||||||
|
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
|
||||||
|
):
|
||||||
|
"""Operator-flagged 2026-06-01: downloaded images stayed at
|
||||||
|
thumbnail_path=NULL until periodic backfill swept them up, surfacing
|
||||||
|
as broken-thumbnail tiles in the gallery. Importer.attach_in_place
|
||||||
|
deliberately skips inline generation; the calling code MUST enqueue
|
||||||
|
the thumbnail + ML tasks per attached image (matching the pattern in
|
||||||
|
tasks/import_file.py)."""
|
||||||
|
from backend.app.services.download_service import DownloadService
|
||||||
|
from backend.app.services.importer import Importer
|
||||||
|
|
||||||
|
_artist, source = seed_artist_and_source
|
||||||
|
|
||||||
|
images_root = tmp_path / "images"
|
||||||
|
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||||
|
f2 = images_root / "alice" / "patreon" / "post" / "b.jpg"
|
||||||
|
_make_jpg(f1, split="h")
|
||||||
|
_make_jpg(f2, split="v")
|
||||||
|
|
||||||
|
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||||
|
success=True, written_paths=[str(f1), str(f2)],
|
||||||
|
files_downloaded=2, stdout=f"{f1}\n{f2}\n",
|
||||||
|
))
|
||||||
|
|
||||||
|
sync_settings = db_sync.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
sync_settings.phash_threshold = 0
|
||||||
|
importer = Importer(
|
||||||
|
session=db_sync, images_root=images_root, import_root=images_root,
|
||||||
|
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||||
|
)
|
||||||
|
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||||
|
|
||||||
|
# Capture the IDs that the orchestrator hands off to each Celery task.
|
||||||
|
# The .delay() shim runs inside DownloadService._phase3_persist (lazy
|
||||||
|
# imports under ..tasks.thumbnail / ..tasks.ml), so monkeypatch the
|
||||||
|
# symbols on those modules.
|
||||||
|
thumb_calls: list[int] = []
|
||||||
|
ml_calls: list[int] = []
|
||||||
|
from backend.app.tasks import ml as ml_mod
|
||||||
|
from backend.app.tasks import thumbnail as thumb_mod
|
||||||
|
monkeypatch.setattr(
|
||||||
|
thumb_mod.generate_thumbnail, "delay",
|
||||||
|
lambda image_id: thumb_calls.append(image_id),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ml_mod.tag_and_embed, "delay",
|
||||||
|
lambda image_id: ml_calls.append(image_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
svc = DownloadService(
|
||||||
|
async_session=db, sync_session=db_sync,
|
||||||
|
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||||
|
)
|
||||||
|
await svc.download_source(source.id)
|
||||||
|
|
||||||
|
# Two files attached → two thumbnail enqueues + two ML enqueues, IDs
|
||||||
|
# match the actually-imported records (not ad-hoc — drawn from the
|
||||||
|
# importer's returned image_id so future supersede paths stay covered).
|
||||||
|
assert len(thumb_calls) == 2
|
||||||
|
assert len(ml_calls) == 2
|
||||||
|
assert sorted(thumb_calls) == sorted(ml_calls)
|
||||||
|
|||||||
Reference in New Issue
Block a user