From 1b25f598c5bfc0fd333269553f90f031b1536b48 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 6 Nov 2026 10:31:04 -0500 Subject: [PATCH] fix: the external-link worker reads the staging-file check from utils, not tasks.maintenance Importing tasks.maintenance made every fetch path reach the membership roster through its sync_memberships task, which test_gated_reason forbids. is_temp_staging_file and its constants now live in utils.staging, imported by both. Co-Authored-By: Claude Opus 5.5 --- backend/app/tasks/external.py | 2 +- backend/app/tasks/maintenance.py | 20 ++++---------------- backend/app/utils/staging.py | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 17 deletions(-) create mode 100644 backend/app/utils/staging.py diff --git a/backend/app/tasks/external.py b/backend/app/tasks/external.py index 43ff98f..92caa2a 100644 --- a/backend/app/tasks/external.py +++ b/backend/app/tasks/external.py @@ -32,8 +32,8 @@ from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, So from ..services.external_fetch import fetch_external from ..services.importer import Importer from ..services.thumbnailer import Thumbnailer +from ..utils.staging import is_temp_staging_file from ._sync_engine import sync_session_factory as _sync_session_factory -from .maintenance import is_temp_staging_file log = logging.getLogger(__name__) diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 1a60b94..3ac6f78 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -24,6 +24,7 @@ from ..models import ( TaskRun, ) from ..utils.phash import compute_phash +from ..utils.staging import is_temp_staging_file from ._sync_engine import get_sync_engine from ._sync_engine import sync_session_factory as _sync_session_factory @@ -81,25 +82,12 @@ WIP_BACKFILL_PAGE = 500 FFPROBE_TIMEOUT_SECONDS = 10 TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days -# Orphaned staging files: downloads/imports stage into .part / .partial -# then os.replace() into place (importer / external_fetch / native_ingest_common / -# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt -# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard -# keeps it from deleting an in-flight download's staging file mid-write. +# Orphaned staging files (utils.staging): cleanup_orphaned_temp_files sweeps the +# ones a killed write left behind; the min-age guard keeps it from deleting an +# in-flight download's staging file mid-write. IMAGES_ROOT = Path("/images") -TEMP_STAGING_SUFFIXES = (".part", ".partial") -# megatools stages a Mega download as `.megatmp.` in the target dir. -TEMP_STAGING_PREFIXES = (".megatmp.",) ORPHAN_TEMP_MIN_AGE_HOURS = 6 - -def is_temp_staging_file(path: Path) -> bool: - """True for a downloader's staging file: one that never becomes library - content under that name, so a stale one is safe to delete.""" - return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith( - TEMP_STAGING_PREFIXES - ) - # Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be # > the entity's longest legitimate runtime (its task's time_limit + a # small buffer) so the sweep never flags in-flight work. diff --git a/backend/app/utils/staging.py b/backend/app/utils/staging.py new file mode 100644 index 0000000..bbeb889 --- /dev/null +++ b/backend/app/utils/staging.py @@ -0,0 +1,20 @@ +"""Downloader staging files. + +Downloads and imports write into a staging name and `os.replace()` it into +place when the write completes (importer, external_fetch, native_ingest_common, +attachment_store), so a write killed midway leaves a staging file, never a +truncated final file. Any other file is complete. +""" +from pathlib import Path + +TEMP_STAGING_SUFFIXES = (".part", ".partial") +# megatools stages a Mega download as `.megatmp.` in the target dir. +TEMP_STAGING_PREFIXES = (".megatmp.",) + + +def is_temp_staging_file(path: Path) -> bool: + """True for a downloader's staging file: one that never becomes library + content under that name, so a stale one is safe to delete.""" + return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith( + TEMP_STAGING_PREFIXES + )