fix: downloads stop leaving files on disk that the library doesn't use
CI and images / lint (push) Successful in 1s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 28s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m25s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 2m3s
CI and images / smoke-web (push) Successful in 55s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped

- An archive whose images were extracted was kept at its download path
  as well as in the attachments store. Both download paths now remove
  the downloaded copy.
- A downloaded file the import filters reject (too small, transparent,
  single colour, unreadable) was left on disk with no record. A later
  walk found it, imported it again and rejected it again, every time.
  Rejected files are now removed; the seen ledger keeps them from being
  downloaded again.
- Both paths share one policy, Importer.discard_after_attach, which
  never removes a file an ImageRecord points at.
- An external-link fetch that fails, or whose worker was killed, left
  partial files (megatools' .megatmp.*) in the link's dir, and a killed
  fetch left its link 'downloading' for good. A fetch now clears
  unrecorded leftovers before starting and after failing. The sweep
  returns a link to retry once it has been 'downloading' past the
  task's hard limit, using a new external_link.started_at column
  (migration 0117).
- The orphaned temp-file sweep now recognises .megatmp.* as well as
  .part and .partial.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 18:00:31 -05:00
co-authored by Claude Opus 5.5
parent 47fb3f2193
commit 157f2803c0
9 changed files with 399 additions and 64 deletions
+63
View File
@@ -193,6 +193,69 @@ async def test_download_source_attaches_written_files(
assert imported_when_marked == [2]
@pytest.mark.asyncio
async def test_import_step_keeps_only_the_files_the_library_needs_on_disk(
db, db_sync, tmp_path, seed_artist_and_source,
):
"""After the import step, a downloaded image the library took stays where it
is, while an archive whose images were extracted (the archive is in the
attachments store) and an image the import filters rejected are removed, so
nothing is left on disk without a record for a later walk to import again."""
import zipfile
from PIL import Image
from backend.app.models import PostAttachment
from backend.app.services.download_service import DownloadService
from backend.app.services.importer import Importer
_artist, source = seed_artist_and_source
images_root = tmp_path / "images"
post_dir = images_root / "alice" / "patreon" / "post"
kept = post_dir / "a.jpg"
_make_jpg(kept, split="h")
member = tmp_path / "member.jpg"
_make_jpg(member, split="v")
archive = post_dir / "pack.zip"
with zipfile.ZipFile(archive, "w") as z:
z.writestr("art.jpg", member.read_bytes())
tiny = post_dir / "tiny.png"
Image.new("RGB", (16, 16), (200, 30, 30)).save(tiny) # below min_width
result = _make_fake_dl_result(
success=True, written_paths=[str(kept), str(archive), str(tiny)],
files_downloaded=3,
)
sync_settings = db_sync.execute(
select(ImportSettings).where(ImportSettings.id == 1)
).scalar_one()
sync_settings.phash_threshold = 0
sync_settings.min_width = 32
importer = Importer(
session=db_sync, images_root=images_root, import_root=images_root,
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
)
crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
svc = DownloadService(
async_session=db, sync_session=db_sync, gdl=_fake_gdl_with_result(result),
importer=importer, cred_service=CredentialService(db, crypto),
)
_stub_patreon_ingester(svc, result)
await svc.download_source(source.id)
assert kept.exists()
assert not archive.exists()
assert not tiny.exists()
# The archive and its image are both still in the library.
assert db_sync.execute(
select(func.count(PostAttachment.id)).where(
PostAttachment.original_filename == "pack.zip"
)
).scalar_one() == 1
assert db_sync.execute(select(func.count(ImageRecord.id))).scalar_one() == 2
@pytest.mark.asyncio
async def test_download_source_skipped_when_disabled(
db, db_sync, tmp_path, seed_artist_and_source,