fix: downloads stop leaving files on disk that the library doesn't use
CI and images / lint (push) Successful in 1s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 28s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m25s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 2m3s
CI and images / smoke-web (push) Successful in 55s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 1s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 28s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m25s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 2m3s
CI and images / smoke-web (push) Successful in 55s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
- An archive whose images were extracted was kept at its download path as well as in the attachments store. Both download paths now remove the downloaded copy. - A downloaded file the import filters reject (too small, transparent, single colour, unreadable) was left on disk with no record. A later walk found it, imported it again and rejected it again, every time. Rejected files are now removed; the seen ledger keeps them from being downloaded again. - Both paths share one policy, Importer.discard_after_attach, which never removes a file an ImageRecord points at. - An external-link fetch that fails, or whose worker was killed, left partial files (megatools' .megatmp.*) in the link's dir, and a killed fetch left its link 'downloading' for good. A fetch now clears unrecorded leftovers before starting and after failing. The sweep returns a link to retry once it has been 'downloading' past the task's hard limit, using a new external_link.started_at column (migration 0117). - The orphaned temp-file sweep now recognises .megatmp.* as well as .part and .partial. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -193,6 +193,69 @@ async def test_download_source_attaches_written_files(
|
||||
assert imported_when_marked == [2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_step_keeps_only_the_files_the_library_needs_on_disk(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""After the import step, a downloaded image the library took stays where it
|
||||
is, while an archive whose images were extracted (the archive is in the
|
||||
attachments store) and an image the import filters rejected are removed, so
|
||||
nothing is left on disk without a record for a later walk to import again."""
|
||||
import zipfile
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from backend.app.models import PostAttachment
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
images_root = tmp_path / "images"
|
||||
post_dir = images_root / "alice" / "patreon" / "post"
|
||||
|
||||
kept = post_dir / "a.jpg"
|
||||
_make_jpg(kept, split="h")
|
||||
member = tmp_path / "member.jpg"
|
||||
_make_jpg(member, split="v")
|
||||
archive = post_dir / "pack.zip"
|
||||
with zipfile.ZipFile(archive, "w") as z:
|
||||
z.writestr("art.jpg", member.read_bytes())
|
||||
tiny = post_dir / "tiny.png"
|
||||
Image.new("RGB", (16, 16), (200, 30, 30)).save(tiny) # below min_width
|
||||
|
||||
result = _make_fake_dl_result(
|
||||
success=True, written_paths=[str(kept), str(archive), str(tiny)],
|
||||
files_downloaded=3,
|
||||
)
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
sync_settings.phash_threshold = 0
|
||||
sync_settings.min_width = 32
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync, gdl=_fake_gdl_with_result(result),
|
||||
importer=importer, cred_service=CredentialService(db, crypto),
|
||||
)
|
||||
_stub_patreon_ingester(svc, result)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
assert kept.exists()
|
||||
assert not archive.exists()
|
||||
assert not tiny.exists()
|
||||
# The archive and its image are both still in the library.
|
||||
assert db_sync.execute(
|
||||
select(func.count(PostAttachment.id)).where(
|
||||
PostAttachment.original_filename == "pack.zip"
|
||||
)
|
||||
).scalar_one() == 1
|
||||
assert db_sync.execute(select(func.count(ImageRecord.id))).scalar_one() == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_source_skipped_when_disabled(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
|
||||
Reference in New Issue
Block a user