feat(deploy): container healthchecks + Swarm rolling-update auto-rollback

web gets a /api/health liveness check; workers a lenient celery-ping check. A
shared deploy policy (update_config order=start-first, failure_action=rollback,
monitor 90s; rollback_config; restart_policy) means a bad image that never goes
healthy is rolled back automatically instead of taking the service down. Ignored
by plain `docker compose up` (deploy: is swarm-only), so the dev override is
unaffected. Assumes prod deploys from this file via docker stack deploy.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/celery_app.py

@@ -115,6 +115,11 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.maintenance.cleanup_old_tasks",
                 "schedule": 86400.0,  # daily
             },
+            "cleanup-orphaned-temp-files": {
+                "task": "backend.app.tasks.maintenance.cleanup_orphaned_temp_files",
+                "schedule": 86400.0,  # daily — sweep .part/.partial left by a
+                # download/import killed mid-write (graceful-shutdown fallout)
+            },
             "train-heads-nightly": {
                 "task": "backend.app.tasks.ml.scheduled_train_heads",
                 "schedule": 86400.0,  # passive cadence; manual retrain stays available

backend/app/tasks/maintenance.py

@@ -79,6 +79,14 @@ VERIFY_PAGE = 200
 FFPROBE_TIMEOUT_SECONDS = 10
 TASK_RUN_KEEP_OK_SECONDS = 24 * 3600       # 24 h
 TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600  # 7 days
+# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
+# then os.replace() into place (importer / external_fetch / native_ingest_common /
+# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
+# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
+# keeps it from deleting an in-flight download's staging file mid-write.
+IMAGES_ROOT = Path("/images")
+TEMP_STAGING_SUFFIXES = (".part", ".partial")
+ORPHAN_TEMP_MIN_AGE_HOURS = 6
 
 # Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
 # > the entity's longest legitimate runtime (its task's time_limit + a
@@ -339,6 +347,34 @@ def cleanup_old_tasks() -> int:
         return result.rowcount or 0
 
 
+@celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
+def cleanup_orphaned_temp_files() -> int:
+    """Delete orphaned .part/.partial staging files under the images root, left by
+    a download/import killed mid-write. Only removes files older than
+    ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
+    pulled out from under it. Returns the count removed."""
+    if not IMAGES_ROOT.is_dir():
+        return 0
+    cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
+    removed = 0
+    for path in IMAGES_ROOT.rglob("*"):
+        if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
+            continue
+        try:
+            if path.stat().st_mtime >= cutoff:
+                continue          # too fresh — may be an active download
+            path.unlink()
+            removed += 1
+        except OSError as exc:
+            log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
+    if removed:
+        log.info(
+            "cleanup_orphaned_temp_files: removed %d orphaned staging file(s)",
+            removed,
+        )
+    return removed
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
 def recover_stalled_task_runs() -> int:
     """Flip task_run rows stuck in 'running' past their queue-specific

docker-compose.yml

@@ -8,6 +8,35 @@
 # run `docker compose up` from this directory and switches images to
 # local builds + DEBUG logging.
 
+# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a
+# time, START the new task before stopping the old (zero-downtime via the ingress
+# mesh), and if the new task doesn't reach a healthy state within `monitor`, roll
+# back to the previous image automatically. `monitor` is sized above web's
+# healthcheck start_period so a broken image that never goes healthy is caught.
+# Plain `docker compose up` ignores `deploy:` (it warns + skips), so the dev
+# override is unaffected. Referenced by each long-lived service below.
+x-deploy-policy: &deploy_policy
+  update_config:
+    order: start-first
+    failure_action: rollback
+    monitor: 90s
+  rollback_config:
+    order: start-first
+  restart_policy:
+    condition: any
+    delay: 10s
+
+# Worker liveness: ping THIS container's celery node over the broker. Lenient
+# (60s interval, 3 retries, 60s start_period) so a transient broker blip never
+# false-flags a worker into a rollback. `$$HOSTNAME` → `$HOSTNAME` for the shell;
+# celery's default node name is celery@<hostname> (the container id).
+x-celery-healthcheck: &celery_healthcheck
+  test: ["CMD-SHELL", "celery -A backend.app.celery_app:celery inspect ping -d celery@$$HOSTNAME --timeout 10 >/dev/null 2>&1"]
+  interval: 60s
+  timeout: 15s
+  retries: 3
+  start_period: 60s
+
 services:
   redis:
     image: redis:7-alpine
@@ -55,6 +84,16 @@ services:
     # by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP
     # requests + the occasional file download.
     stop_grace_period: 30s
+    # Liveness for rolling deploys: /api/health is a no-DB 200 (just proves the
+    # app booted + serves HTTP after `alembic upgrade head`). start_period covers
+    # the migration + boot so a slow start isn't mis-flagged.
+    healthcheck:
+      test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/api/health', timeout=5).status==200 else 1)\""]
+      interval: 15s
+      timeout: 6s
+      retries: 3
+      start_period: 40s
+    deploy: *deploy_policy
     ports:
       - "${PORT:-8080}:8080"
     environment: &app_env
@@ -87,6 +126,8 @@ services:
     command: ["worker"]
     # Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
     stop_grace_period: 90s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
       CELERY_QUEUES: default,import,thumbnail,download
@@ -105,6 +146,8 @@ services:
     command: ["scheduler"]
     # Quick maintenance/scan lane + beat — short tasks, modest drain window.
     stop_grace_period: 60s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
       CELERY_QUEUES: maintenance,scan
@@ -126,6 +169,8 @@ services:
     # the most room to finish a chunk gracefully. Chunked + idempotent, so a job
     # that still outruns this resumes cleanly next run rather than corrupting.
     stop_grace_period: 180s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
       CELERY_QUEUES: maintenance_long
@@ -143,6 +188,8 @@ services:
     command: ["ml-worker"]
     # A single GPU inference pass can run tens of seconds — let it finish.
     stop_grace_period: 120s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
     volumes:

tests/test_maintenance.py

@@ -26,6 +26,28 @@ def _make_batch(session) -> int:
     return batch.id
 
 
+def test_cleanup_orphaned_temp_files_removes_stale_only(tmp_path, monkeypatch):
+    import os
+
+    from backend.app.tasks import maintenance as m
+
+    monkeypatch.setattr(m, "IMAGES_ROOT", tmp_path)
+    stale = tmp_path / "artist" / "img.jpg.part"      # killed download → orphan
+    stale.parent.mkdir(parents=True)
+    stale.write_bytes(b"x")
+    old = datetime.now(UTC).timestamp() - 8 * 3600     # older than the 6h guard
+    os.utime(stale, (old, old))
+    fresh = tmp_path / "in_progress.jpg.partial"       # active download → keep
+    fresh.write_bytes(b"x")
+    keep = tmp_path / "real.jpg"                        # a real image → keep
+    keep.write_bytes(b"x")
+
+    assert m.cleanup_orphaned_temp_files() == 1
+    assert not stale.exists()
+    assert fresh.exists()
+    assert keep.exists()
+
+
 def test_recover_interrupted_only_old(db_sync, monkeypatch):
     batch_id = _make_batch(db_sync)
     now = datetime.now(UTC)