fix: downloads stop leaving files on disk that the library doesn't use

- An archive whose images were extracted was kept at its download path
  as well as in the attachments store. Both download paths now remove
  the downloaded copy.
- A downloaded file the import filters reject (too small, transparent,
  single colour, unreadable) was left on disk with no record. A later
  walk found it, imported it again and rejected it again, every time.
  Rejected files are now removed; the seen ledger keeps them from being
  downloaded again.
- Both paths share one policy, Importer.discard_after_attach, which
  never removes a file an ImageRecord points at.
- An external-link fetch that fails, or whose worker was killed, left
  partial files (megatools' .megatmp.*) in the link's dir, and a killed
  fetch left its link 'downloading' for good. A fetch now clears
  unrecorded leftovers before starting and after failing. The sweep
  returns a link to retry once it has been 'downloading' past the
  task's hard limit, using a new external_link.started_at column
  (migration 0117).
- The orphaned temp-file sweep now recognises .megatmp.* as well as
  .part and .partial.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_download_service.py

@@ -193,6 +193,69 @@ async def test_download_source_attaches_written_files(
     assert imported_when_marked == [2]
 
 
+@pytest.mark.asyncio
+async def test_import_step_keeps_only_the_files_the_library_needs_on_disk(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """After the import step, a downloaded image the library took stays where it
+    is, while an archive whose images were extracted (the archive is in the
+    attachments store) and an image the import filters rejected are removed, so
+    nothing is left on disk without a record for a later walk to import again."""
+    import zipfile
+
+    from PIL import Image
+
+    from backend.app.models import PostAttachment
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    images_root = tmp_path / "images"
+    post_dir = images_root / "alice" / "patreon" / "post"
+
+    kept = post_dir / "a.jpg"
+    _make_jpg(kept, split="h")
+    member = tmp_path / "member.jpg"
+    _make_jpg(member, split="v")
+    archive = post_dir / "pack.zip"
+    with zipfile.ZipFile(archive, "w") as z:
+        z.writestr("art.jpg", member.read_bytes())
+    tiny = post_dir / "tiny.png"
+    Image.new("RGB", (16, 16), (200, 30, 30)).save(tiny)  # below min_width
+
+    result = _make_fake_dl_result(
+        success=True, written_paths=[str(kept), str(archive), str(tiny)],
+        files_downloaded=3,
+    )
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    sync_settings.phash_threshold = 0
+    sync_settings.min_width = 32
+    importer = Importer(
+        session=db_sync, images_root=images_root, import_root=images_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
+    )
+    crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync, gdl=_fake_gdl_with_result(result),
+        importer=importer, cred_service=CredentialService(db, crypto),
+    )
+    _stub_patreon_ingester(svc, result)
+    await svc.download_source(source.id)
+
+    assert kept.exists()
+    assert not archive.exists()
+    assert not tiny.exists()
+    # The archive and its image are both still in the library.
+    assert db_sync.execute(
+        select(func.count(PostAttachment.id)).where(
+            PostAttachment.original_filename == "pack.zip"
+        )
+    ).scalar_one() == 1
+    assert db_sync.execute(select(func.count(ImageRecord.id))).scalar_one() == 2
+
+
 @pytest.mark.asyncio
 async def test_download_source_skipped_when_disabled(
     db, db_sync, tmp_path, seed_artist_and_source,

tests/test_external_worker.py

@@ -251,3 +251,154 @@ def test_downloaded_archive_gets_provenance_and_tagging(db_sync, tmp_path, monke
     # Tagging + thumbnails queued for exactly those member images.
     assert set(tagged) == member_ids
     assert set(thumbed) == member_ids
+
+
+def _link_dir(tmp_path, host, link_id):
+    return tmp_path / f"ext-{host}" / "patreon" / "external" / f"EXT-{host}" / str(link_id)
+
+
+def test_extracted_archive_is_not_left_in_the_link_dir(db_sync, tmp_path, monkeypatch):
+    """An archive whose images were extracted is in the attachments store, so
+    the downloaded copy is removed instead of sitting beside it."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    written = []
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        zpath = dest_dir / "pack.zip"
+        with zipfile.ZipFile(zpath, "w") as z:
+            z.writestr("art.jpg", _structured_jpeg_bytes())
+        written.append(zpath)
+        return FetchResult(files=[zpath], bytes=zpath.stat().st_size)
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    import backend.app.tasks.ml as ml_mod
+    import backend.app.tasks.thumbnail as thumb_mod
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
+    monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
+
+    out = ext.fetch_external_link(link_id)
+    assert out["images"] >= 1
+    assert not written[0].exists()
+    db_sync.expire_all()
+    assert db_sync.execute(
+        select(func.count(PostAttachment.id))
+        .where(PostAttachment.original_filename == "pack.zip")
+    ).scalar_one() == 1
+
+
+def test_a_failed_fetch_leaves_no_partial_files(db_sync, tmp_path, monkeypatch):
+    """A fetch that fails midway (megatools leaves `.megatmp.*` behind) clears
+    the link's staging dir, so a dead link leaves nothing on disk."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        (dest_dir / ".megatmp.AbCd").write_bytes(b"half a game build")
+        return FetchResult(error="megatools dl exit 1: timed out")
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    ext.fetch_external_link(link_id)
+
+    link_dir = _link_dir(tmp_path, "mega", link_id)
+    assert not (link_dir / ".megatmp.AbCd").exists()
+    db_sync.expire_all()
+    assert db_sync.get(ExternalLink, link_id).status == "failed"
+
+
+def test_a_retry_clears_what_a_killed_attempt_left(db_sync, tmp_path, monkeypatch):
+    """Leftovers from an attempt whose worker was killed are removed before the
+    next fetch starts; an image imported in place by an earlier attempt stays."""
+    _, link = _seed(db_sync, host="pixeldrain")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    link_dir = _link_dir(tmp_path, "pixeldrain", link_id)
+    link_dir.mkdir(parents=True)
+    leftover = link_dir / "build.zip.part"
+    leftover.write_bytes(b"partial")
+    imported = link_dir / "kept.jpg"
+    imported.write_bytes(_structured_jpeg_bytes())
+    db_sync.add(ImageRecord(
+        path=str(imported), sha256="ext-kept".ljust(64, "x"),
+        size_bytes=imported.stat().st_size, mime="image/jpeg",
+        width=256, height=256, origin="downloaded", integrity_status="ok",
+    ))
+    db_sync.commit()
+    seen_at_fetch = []
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        seen_at_fetch.extend(sorted(p.name for p in dest_dir.iterdir()))
+        return FetchResult(error="HTTP 503")
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    ext.fetch_external_link(link_id)
+
+    assert seen_at_fetch == ["kept.jpg"]
+    assert imported.exists()
+    assert not leftover.exists()
+
+
+def test_a_rejected_download_is_removed(db_sync, tmp_path, monkeypatch):
+    """A downloaded image the import filters reject has no record, so it is
+    removed rather than left in the link's dir."""
+    from backend.app.models import ImportSettings
+
+    _, link = _seed(db_sync, host="dropbox")
+    link_id = link.id
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    settings.min_width = 512
+    db_sync.commit()
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    written = []
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        f = dest_dir / "small.jpg"  # 256px wide, below min_width
+        f.write_bytes(_structured_jpeg_bytes())
+        written.append(f)
+        return FetchResult(files=[f], bytes=f.stat().st_size)
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    out = ext.fetch_external_link(link_id)
+
+    assert out["images"] == 0
+    assert not written[0].exists()
+    db_sync.expire_all()
+    assert db_sync.get(ExternalLink, link_id).status == "downloaded"
+
+
+def test_sweep_recovers_a_link_whose_worker_died(db_sync, monkeypatch):
+    """A link 'downloading' past the task's hard limit (or with no claim time)
+    counts as a failed attempt and is retried; a fresh claim is left alone."""
+    from datetime import UTC, datetime, timedelta
+
+    _, stale = _seed(db_sync, host="mega", status="downloading")
+    _, unclaimed = _seed(db_sync, host="gdrive", status="downloading")
+    _, fresh = _seed(db_sync, host="dropbox", status="downloading")
+    now = datetime.now(UTC)
+    stale.started_at = now - timedelta(minutes=ext._STALL_MINUTES + 5)
+    fresh.started_at = now - timedelta(minutes=5)
+    db_sync.commit()
+    stale_id, unclaimed_id, fresh_id = stale.id, unclaimed.id, fresh.id
+
+    enqueued = []
+    monkeypatch.setattr(ext.fetch_external_link, "delay", lambda lid: enqueued.append(lid))
+    out = ext.sweep_external_links()
+
+    assert out["recovered"] == 2
+    assert set(enqueued) == {stale_id, unclaimed_id}
+    db_sync.expire_all()
+    for lid in (stale_id, unclaimed_id):
+        row = db_sync.get(ExternalLink, lid)
+        assert (row.status, row.attempts) == ("failed", 1)
+    assert db_sync.get(ExternalLink, fresh_id).status == "downloading"

tests/test_maintenance.py

@@ -41,9 +41,13 @@ def test_cleanup_orphaned_temp_files_removes_stale_only(tmp_path, monkeypatch):
     fresh.write_bytes(b"x")
     keep = tmp_path / "real.jpg"                        # a real image → keep
     keep.write_bytes(b"x")
+    mega = tmp_path / "artist" / ".megatmp.AbCd123"     # abandoned megatools stage
+    mega.write_bytes(b"x")
+    os.utime(mega, (old, old))
 
-    assert m.cleanup_orphaned_temp_files() == 1
+    assert m.cleanup_orphaned_temp_files() == 2
     assert not stale.exists()
+    assert not mega.exists()
     assert fresh.exists()
     assert keep.exists()