fix: an unfinished download is never stored as an attachment, and the temp sweep keeps any file a library row points at
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 24s
CI and images / frontend-build (push) Successful in 26s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m23s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m59s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 24s
CI and images / frontend-build (push) Successful in 26s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m23s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m59s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The importer refuses a staging file (.part, .partial, .megatmp.*) at both entry points, the same way it refuses a sidecar .json, and discard_after_attach never deletes one: the age-gated temp sweep owns them. The sweep checks its stale candidates against image and attachment rows and keeps any a row points at. A Google Drive fetch's half-written file had been stored as <sha>.part; the sweep then deleted it by name and left the row with no file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from ..models import (
|
||||
ImportSettings,
|
||||
ImportTask,
|
||||
LibraryAuditRun,
|
||||
PostAttachment,
|
||||
Source,
|
||||
TaskRun,
|
||||
)
|
||||
@@ -372,23 +373,58 @@ def cleanup_old_tasks() -> int:
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def _paths_with_rows(paths: list[str]) -> set[str]:
|
||||
"""The subset of `paths` that an image (its file or thumbnail) or an
|
||||
attachment row points at."""
|
||||
owned: set[str] = set()
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
for i in range(0, len(paths), 500):
|
||||
chunk = paths[i:i + 500]
|
||||
owned.update(session.execute(
|
||||
select(ImageRecord.path).where(ImageRecord.path.in_(chunk))
|
||||
).scalars())
|
||||
owned.update(session.execute(
|
||||
select(ImageRecord.thumbnail_path).where(ImageRecord.thumbnail_path.in_(chunk))
|
||||
).scalars())
|
||||
owned.update(session.execute(
|
||||
select(PostAttachment.path).where(PostAttachment.path.in_(chunk))
|
||||
).scalars())
|
||||
return owned
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
|
||||
def cleanup_orphaned_temp_files() -> int:
|
||||
"""Delete orphaned staging files (.part, .partial, megatools' .megatmp.*)
|
||||
under the images root, left by a download/import killed mid-write. Only
|
||||
removes files older than ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight
|
||||
download's staging file is never pulled out from under it. Returns the
|
||||
count removed."""
|
||||
download's staging file is never pulled out from under it, and never a
|
||||
file an image or attachment row points at, whatever its name: that file
|
||||
is library content. Returns the count removed."""
|
||||
if not IMAGES_ROOT.is_dir():
|
||||
return 0
|
||||
cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
|
||||
removed = 0
|
||||
stale: list[Path] = []
|
||||
for path in IMAGES_ROOT.rglob("*"):
|
||||
if not is_temp_staging_file(path) or not path.is_file():
|
||||
continue
|
||||
try:
|
||||
if path.stat().st_mtime >= cutoff:
|
||||
continue # too fresh — may be an active download
|
||||
if path.stat().st_mtime < cutoff: # fresher may be an active download
|
||||
stale.append(path)
|
||||
except OSError as exc:
|
||||
log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
|
||||
if not stale:
|
||||
return 0
|
||||
owned = _paths_with_rows([str(p) for p in stale])
|
||||
removed = 0
|
||||
for path in stale:
|
||||
if str(path) in owned:
|
||||
log.warning(
|
||||
"cleanup_orphaned_temp_files: keeping %s: a library row points at it",
|
||||
path,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError as exc:
|
||||
|
||||
Reference in New Issue
Block a user