fix: the external-link worker reads the staging-file check from utils, not tasks.maintenance
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / backend-lint-and-test (push) Successful in 36s
CI and images / frontend-build (push) Successful in 26s
CI and images / extension-test (push) Successful in 24s
CI and images / integration (push) Successful in 2m34s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m47s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 2s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / backend-lint-and-test (push) Successful in 36s
CI and images / frontend-build (push) Successful in 26s
CI and images / extension-test (push) Successful in 24s
CI and images / integration (push) Successful in 2m34s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m47s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 2s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
Importing tasks.maintenance made every fetch path reach the membership roster through its sync_memberships task, which test_gated_reason forbids. is_temp_staging_file and its constants now live in utils.staging, imported by both. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -32,8 +32,8 @@ from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, So
|
|||||||
from ..services.external_fetch import fetch_external
|
from ..services.external_fetch import fetch_external
|
||||||
from ..services.importer import Importer
|
from ..services.importer import Importer
|
||||||
from ..services.thumbnailer import Thumbnailer
|
from ..services.thumbnailer import Thumbnailer
|
||||||
|
from ..utils.staging import is_temp_staging_file
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
from .maintenance import is_temp_staging_file
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ from ..models import (
|
|||||||
TaskRun,
|
TaskRun,
|
||||||
)
|
)
|
||||||
from ..utils.phash import compute_phash
|
from ..utils.phash import compute_phash
|
||||||
|
from ..utils.staging import is_temp_staging_file
|
||||||
from ._sync_engine import get_sync_engine
|
from ._sync_engine import get_sync_engine
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
@@ -81,25 +82,12 @@ WIP_BACKFILL_PAGE = 500
|
|||||||
FFPROBE_TIMEOUT_SECONDS = 10
|
FFPROBE_TIMEOUT_SECONDS = 10
|
||||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
|
# Orphaned staging files (utils.staging): cleanup_orphaned_temp_files sweeps the
|
||||||
# then os.replace() into place (importer / external_fetch / native_ingest_common /
|
# ones a killed write left behind; the min-age guard keeps it from deleting an
|
||||||
# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
|
# in-flight download's staging file mid-write.
|
||||||
# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
|
|
||||||
# keeps it from deleting an in-flight download's staging file mid-write.
|
|
||||||
IMAGES_ROOT = Path("/images")
|
IMAGES_ROOT = Path("/images")
|
||||||
TEMP_STAGING_SUFFIXES = (".part", ".partial")
|
|
||||||
# megatools stages a Mega download as `.megatmp.<id>` in the target dir.
|
|
||||||
TEMP_STAGING_PREFIXES = (".megatmp.",)
|
|
||||||
ORPHAN_TEMP_MIN_AGE_HOURS = 6
|
ORPHAN_TEMP_MIN_AGE_HOURS = 6
|
||||||
|
|
||||||
|
|
||||||
def is_temp_staging_file(path: Path) -> bool:
|
|
||||||
"""True for a downloader's staging file: one that never becomes library
|
|
||||||
content under that name, so a stale one is safe to delete."""
|
|
||||||
return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith(
|
|
||||||
TEMP_STAGING_PREFIXES
|
|
||||||
)
|
|
||||||
|
|
||||||
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
||||||
# > the entity's longest legitimate runtime (its task's time_limit + a
|
# > the entity's longest legitimate runtime (its task's time_limit + a
|
||||||
# small buffer) so the sweep never flags in-flight work.
|
# small buffer) so the sweep never flags in-flight work.
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
"""Downloader staging files.
|
||||||
|
|
||||||
|
Downloads and imports write into a staging name and `os.replace()` it into
|
||||||
|
place when the write completes (importer, external_fetch, native_ingest_common,
|
||||||
|
attachment_store), so a write killed midway leaves a staging file, never a
|
||||||
|
truncated final file. Any other file is complete.
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
TEMP_STAGING_SUFFIXES = (".part", ".partial")
|
||||||
|
# megatools stages a Mega download as `.megatmp.<id>` in the target dir.
|
||||||
|
TEMP_STAGING_PREFIXES = (".megatmp.",)
|
||||||
|
|
||||||
|
|
||||||
|
def is_temp_staging_file(path: Path) -> bool:
|
||||||
|
"""True for a downloader's staging file: one that never becomes library
|
||||||
|
content under that name, so a stale one is safe to delete."""
|
||||||
|
return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith(
|
||||||
|
TEMP_STAGING_PREFIXES
|
||||||
|
)
|
||||||
Reference in New Issue
Block a user