fix: an unfinished download is never stored as an attachment, and the temp sweep keeps any file a library row points at
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 24s
CI and images / frontend-build (push) Successful in 26s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m23s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m59s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 24s
CI and images / frontend-build (push) Successful in 26s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m23s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m59s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The importer refuses a staging file (.part, .partial, .megatmp.*) at both entry points, the same way it refuses a sidecar .json, and discard_after_attach never deletes one: the age-gated temp sweep owns them. The sweep checks its stale candidates against image and attachment rows and keeps any a row points at. A Google Drive fetch's half-written file had been stored as <sha>.part; the sweep then deleted it by name and left the row with no file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -410,6 +410,31 @@ def test_attach_in_place_invalid_image_returns_skipped(importer):
|
||||
assert result.skip_reason.value == "invalid_image"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["film.mp4abc123_.part", "img.jpg.partial", ".megatmp.Xy12"])
|
||||
def test_an_unfinished_download_is_refused_and_left_on_disk(importer, db_sync, name):
|
||||
"""A staging file is a download still being written (or abandoned): it is
|
||||
never stored as an attachment, and the importer never deletes it, so a
|
||||
download in progress is not pulled out from under its writer."""
|
||||
from backend.app.models import PostAttachment
|
||||
|
||||
artist = Artist(name="Hal", slug="hal")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
staging = importer.images_root / "hal" / "patreon" / "post" / name
|
||||
staging.parent.mkdir(parents=True, exist_ok=True)
|
||||
staging.write_bytes(b"half of a video")
|
||||
|
||||
for result in (importer.attach_in_place(staging, artist=artist),
|
||||
importer.import_one(staging)):
|
||||
assert result.status == "skipped"
|
||||
assert "unfinished download" in result.error
|
||||
assert importer.discard_after_attach(staging, result) is False
|
||||
assert staging.exists()
|
||||
assert db_sync.execute(
|
||||
select(func.count()).where(PostAttachment.original_filename == name)
|
||||
).scalar() == 0
|
||||
|
||||
|
||||
def test_attach_in_place_non_media_routes_to_attachment(importer, db_sync):
|
||||
"""FC-2d-iii dispatch parity for the download path. A non-media file
|
||||
(.pdf, .txt etc.) downloaded by gallery-dl must become a PostAttachment,
|
||||
|
||||
+31
-11
@@ -26,30 +26,50 @@ def _make_batch(session) -> int:
|
||||
return batch.id
|
||||
|
||||
|
||||
def test_cleanup_orphaned_temp_files_removes_stale_only(tmp_path, monkeypatch):
|
||||
def test_cleanup_orphaned_temp_files_removes_stale_only(db_sync, tmp_path, monkeypatch):
|
||||
import os
|
||||
|
||||
from backend.app.models import ImageRecord, PostAttachment
|
||||
from backend.app.tasks import maintenance as m
|
||||
|
||||
monkeypatch.setattr(m, "IMAGES_ROOT", tmp_path)
|
||||
stale = tmp_path / "artist" / "img.jpg.part" # killed download → orphan
|
||||
stale.parent.mkdir(parents=True)
|
||||
stale.write_bytes(b"x")
|
||||
old = datetime.now(UTC).timestamp() - 8 * 3600 # older than the 6h guard
|
||||
os.utime(stale, (old, old))
|
||||
fresh = tmp_path / "in_progress.jpg.partial" # active download → keep
|
||||
fresh.write_bytes(b"x")
|
||||
keep = tmp_path / "real.jpg" # a real image → keep
|
||||
|
||||
def staged(rel: str, *, aged: bool = True):
|
||||
p = tmp_path / rel
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(b"x")
|
||||
if aged:
|
||||
os.utime(p, (old, old))
|
||||
return p
|
||||
|
||||
stale = staged("artist/img.jpg.part") # killed download → orphan
|
||||
mega = staged("artist/.megatmp.AbCd123") # abandoned megatools stage
|
||||
fresh = staged("in_progress.jpg.partial", aged=False) # active download → keep
|
||||
keep = tmp_path / "real.jpg" # a real image → keep
|
||||
keep.write_bytes(b"x")
|
||||
mega = tmp_path / "artist" / ".megatmp.AbCd123" # abandoned megatools stage
|
||||
mega.write_bytes(b"x")
|
||||
os.utime(mega, (old, old))
|
||||
# Staging-named files a library row points at are library content.
|
||||
attached = staged("attachments/ab/abc.part")
|
||||
imaged = staged("artist/video.mp4.part")
|
||||
db_sync.add(PostAttachment(
|
||||
post_id=None, artist_id=None, sha256="tempsweep-att".ljust(64, "x"),
|
||||
path=str(attached), original_filename="film.mp4.part", ext=".part",
|
||||
size_bytes=1,
|
||||
))
|
||||
db_sync.add(ImageRecord(
|
||||
path=str(imaged), sha256="tempsweep-img".ljust(64, "x"), size_bytes=1,
|
||||
mime="video/mp4", width=1, height=1, origin="downloaded",
|
||||
integrity_status="ok",
|
||||
))
|
||||
db_sync.commit()
|
||||
|
||||
assert m.cleanup_orphaned_temp_files() == 2
|
||||
assert not stale.exists()
|
||||
assert not mega.exists()
|
||||
assert fresh.exists()
|
||||
assert keep.exists()
|
||||
assert attached.exists()
|
||||
assert imaged.exists()
|
||||
|
||||
|
||||
def test_recover_interrupted_only_old(db_sync, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user