style: sort two import blocks the download cleanup left out of order

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.env.example

@@ -56,9 +56,8 @@ SECRET_KEY=
 # LINE. Leaving it set disarms the protection permanently, on an instance that
 # by then has credentials worth protecting.
 #
-# BACK UP ./images/secrets/ ALONGSIDE YOUR DATABASE. The key is the only thing
-# that can read your stored credentials; a database restored without it needs
-# every credential re-entered by hand.
+# Backups leave the key and the stored logins out on purpose: after restoring
+# onto a new install, sign in to each account again.
 CURATOR_BOOTSTRAP_NEW_KEY=1
 
 

.forgejo/workflows/baseline.yml

@@ -81,8 +81,18 @@ jobs:
           set -eux
           # Same service-IP dance as build.yml's integration job; see the long
           # comment there for why the job name must stay separator-free.
-          PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          test -n "$PG"
+          # Only THIS job's services. The runner's docker daemon is shared, so another
+          # repo's job can be running beside this one, and a job-name filter matches its
+          # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
+          # Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
+          # <n> from our own container (its id is in the /etc/hostname bind mount) and
+          # require exactly one match.
+          SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
+          SELF=${SELF:-$(hostname)}
+          TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
+          test -n "$TASK"
+          PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
+          test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           test -n "$PG_IP"
           echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"

.forgejo/workflows/build.yml

@@ -142,6 +142,16 @@ concurrency:
 # Deriving it per job invites the two halves to disagree: sign-extension would
 # derive dev's extension version while build-web bundled main's, and the
 # release download would 404 on a version that exists perfectly well.
+#
+# On every other trigger it is the COMMIT that fired (`github.sha`), never the
+# branch name. A branch is re-resolved by each job's checkout when that job
+# starts, so a push landing mid-run moved the later jobs onto the new tip: run
+# 7499 signed 423275a's extension, then build-web checked out 83e1382 (pushed
+# while 7499 ran), derived a version nobody had signed, and 404'd on the
+# download (#4427). The guard failed closed that time; a job without one would
+# have published a commit the run's own lanes never tested — the lanes check
+# out `github.sha` by default, so pinning here makes the publish build exactly
+# what they passed.
 # IS THIS A BASE REFRESH? Asked in five places and previously spelled five
 # ways — `github.event_name == 'schedule'` in an `if:`, `$GITHUB_EVENT_NAME` in
 # one shell, an `EVENT:` env passed into another, and a bare expression on
@@ -178,12 +188,22 @@ concurrency:
 # where a step-level `if:` needs the answer before any shell runs.
 env:
   IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }}
-  BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.ref }}
+  BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.sha }}
+# What the six LANES check out. Empty — the checkout default, the triggering
+# commit (or a PR's merge ref) — on every trigger but the refresh, where it is
+# `main`: the refresh publishes main, so the gate has to test main (#4430). It
+# is not BUILD_REF itself because a pull_request run's `github.sha` is a merge
+# commit the default checkout reaches through its ref, not by sha. Each job
+# still resolves `main` when it starts, so a merge to main during the ~5 min of
+# a Sunday-06:00 refresh could put the lanes and the build one commit apart;
+# the build jobs' own guards assert the branch, not the commit.
+  LANE_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || '' }}
 
 # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
 #   - write:package, read:package  (for docker push to git.fabledsword.com)
 #   - write:release                (for ext-<version> release asset cache)
-#   - write:issue                  (for future issue-management automation)
+#   - write:issue                  (`unattended-failure` opens an issue)
+#   - write:repository             (`refresh-dev` dispatches this workflow)
 # The injected GITHUB_TOKEN cannot be used — it lacks write:package.
 
 jobs:
@@ -219,6 +239,8 @@ jobs:
       image: git.fabledsword.com/bvandeusen/ci-python:3.14
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ env.LANE_REF }}
       - name: Ruff lint
         # agent/ included so the GPU-agent is linted before its image is built
         # (build.yml only `docker build`s it — this is where it gets checked).
@@ -242,7 +264,7 @@ jobs:
   # rather than left running beside the new mechanism (rule 22).
   #
   # Two things are still worth asserting, and this is the only lane that can:
-  # the extension.yml suite runs on node:24-slim, which is exactly why
+  # the extension-test lane runs on node:24-slim, which is exactly why
   # version.spec.js sticks to packaging.sh's git-free subcommands.
   #   1. the derivation actually resolves on this commit
   #   2. the derived string is one AMO will accept, checked against Mozilla's
@@ -265,6 +287,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          ref: ${{ env.LANE_REF }}
           # The derivation needs real history: a depth-1 clone sees one commit
           # and produces a wrong, too-low value RATHER THAN FAILING. Checking
           # that here is half the point of the lane.
@@ -320,6 +343,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          ref: ${{ env.LANE_REF }}
           # Full history for tests/test_artifact_identity.py, which derives
           # each artifact's revision to check the identity scheme. On a
           # depth-1 clone that derivation either fails or returns the tip sha
@@ -366,6 +390,8 @@ jobs:
         working-directory: frontend
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ env.LANE_REF }}
       # No package-lock.json is tracked yet (we don't run npm locally per
       # feedback-no-local-runs). Using `npm install` instead of `npm ci`.
       # If we want strict lockfile-based reproducibility later, commit a
@@ -378,6 +404,78 @@ jobs:
       - run: npm run test:unit
       - run: npm run build
 
+  # The extension's lane: web-ext lint, the vitest suite, and the check that
+  # asks web-ext what it ACTUALLY packaged. Moved in from `extension.yml`
+  # (deleted) at milestone 429 — the same move `ci.yml` made on 2026-09-23 and
+  # for the same reason: a separate workflow cannot gate this one, so a red
+  # extension suite still let `sign-extension` sign and `build-web` ship the
+  # XPI. It is a lane in THE GATE now. Its old path filter is gone with it; a
+  # filter here would make the lane skip, and a skipped lane blocks the publish.
+  #
+  # The vitest suite is also the JS half of the #3093 artist-pattern mirror —
+  # the Python half runs in backend-lint-and-test — so until this move one half
+  # of that guard could fail without stopping anything.
+  extension-test:
+    runs-on: python-ci
+    container:
+      image: node:24-bookworm-slim
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ env.LANE_REF }}
+      # Not --no-save: vitest and web-ext are both real devDependencies now,
+      # and the suite needs vitest resolvable from node_modules.
+      - name: Install dev dependencies
+        run: cd extension && npm install --no-audit --no-fund
+      - name: Lint
+        run: cd extension && npm run lint
+      # Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
+      # package version-consistency checks. No browser, no network.
+      - name: Unit tests
+        run: cd extension && npm run test:unit
+
+      # Everything else about packaging is asserted against our own declaration
+      # of what ships. This is the only check that asks web-ext what it ACTUALLY
+      # put in the archive. Until now that was an unverified assumption about
+      # glob semantics — and a fragile one: `test/**` reaches web-ext intact
+      # only because callers `set -f` first, so losing that quoting would
+      # silently start shipping dev files with no other signal.
+      - name: Verify XPI contents
+        run: |
+          set -eu
+          command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
+          cd extension
+          npm run build
+          ZIP=$(ls web-ext-artifacts/*.zip | head -1)
+          echo "=== packaged entries in $ZIP ==="
+          unzip -Z1 "$ZIP" | sort
+          echo "=== end ==="
+          ENTRIES=$(unzip -Z1 "$ZIP")
+          fail=0
+          # Must NOT ship: repo infrastructure with no business in a user's browser.
+          for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
+            if echo "$ENTRIES" | grep -q "^$pat"; then
+              echo "ERROR: '$pat' was packaged into the XPI but must not be"
+              fail=1
+            fi
+          done
+          # Must ship: if an exclusion pattern ever over-matches, the extension
+          # breaks at runtime rather than at build time, so assert presence too.
+          for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js' 'lib/chip.js' 'lib/popup-format.js'; do
+            if ! echo "$ENTRIES" | grep -q "^$req$"; then
+              echo "ERROR: '$req' is missing from the XPI"
+              fail=1
+            fi
+          done
+          for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
+            if ! echo "$ENTRIES" | grep -q "^$dir"; then
+              echo "ERROR: nothing from '$dir' was packaged"
+              fail=1
+            fi
+          done
+          [ "$fail" -eq 0 ] || exit 1
+          echo "XPI contents verified."
+
   # Single integration job — collapsed from a 3-way shard split on 2026-06-04.
   # The shards existed to parallelize ~8.5min of integration tests; once the
   # throwaway Postgres runs with fsync OFF (the durability step below) the whole
@@ -419,7 +517,7 @@ jobs:
           --health-timeout 5s
           --health-retries 10
       redis:
-        image: redis:7-alpine
+        image: redis:8-alpine
         options: >-
           --health-cmd "redis-cli ping"
           --health-interval 10s
@@ -427,15 +525,28 @@ jobs:
           --health-retries 10
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ env.LANE_REF }}
       - name: Integration suite (resolve service IPs, migrate, test)
         run: |
           set -eux
           echo "=== container landscape (diagnostic for filter scoping) ==="
           docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
           echo "=== end landscape ==="
-          PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
-          test -n "$PG" && test -n "$RD"
+          # Only THIS job's services. The runner's docker daemon is shared, so another
+          # repo's job can be running beside this one, and a job-name filter matches its
+          # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
+          # Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
+          # <n> from our own container (its id is in the /etc/hostname bind mount) and
+          # require exactly one match.
+          SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
+          SELF=${SELF:-$(hostname)}
+          TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
+          test -n "$TASK"
+          PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
+          test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
+          RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
+          test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
           test -n "$PG_IP" && test -n "$RD_IP"
@@ -468,6 +579,9 @@ jobs:
           else
             pip install -r requirements.txt pytest pytest-asyncio
           fi
+          # The backup round trip runs the real pg_dump and pg_restore, from the
+          # same Debian client package the app image installs.
+          command -v pg_dump >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq postgresql-client; }
           # Relax durability on the throwaway CI Postgres so the per-test
           # TRUNCATE's commit-fsync — the integration teardown's dominant cost
           # (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is
@@ -521,7 +635,8 @@ jobs:
     # itself through a job-level `if:` that could not read `env`, and a design
     # where only a FAILED gate blocks would have published unverified images
     # while reporting success.
-    needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
+    needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
+            integration]
     # A pull_request run is the lanes and nothing else. This is the ONLY thing
     # separating "validate a Renovate bump" from "publish a Renovate bump", so
     # it is stated on each publishing job rather than inferred from a `needs`
@@ -534,8 +649,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           # Not the triggering ref — see the `env:` block at the top. On a
-          # scheduled refresh this is `main`; on everything else it is the ref
-          # that fired, so this is a no-op on every ordinary path.
+          # scheduled refresh this is `main`; on everything else it is the
+          # commit that fired, the same one the lanes above tested.
           ref: ${{ env.BUILD_REF }}
           # Full history is load-bearing, not a convenience: the version this
           # job signs is derived from the commit TIME of the newest packaged
@@ -676,6 +791,15 @@ jobs:
               print("guard: no ext-* release yet — nothing to go backwards from")
               raise SystemExit(0)
 
+          # A version that is already signed is reused from its release, not
+          # signed again, so it cannot be a downgrade. The weekly refresh builds
+          # main, whose version trails whatever dev has signed since; refusing
+          # it here would stop :latest from ever being refreshed while dev is
+          # ahead.
+          if any(v == derived for v, _ in signed):
+              print(f"guard: {derived_s} is already signed — reused, not re-signed")
+              raise SystemExit(0)
+
           hi, hi_tag = signed[-1]
           print(f"guard: derived={derived_s}  highest already signed={hi_tag}")
           if derived < hi:
@@ -850,13 +974,16 @@ jobs:
       # reuse hit — deliberately NOT folded into `digest`, which the :c-<sha>
       # repoint reads and which must keep meaning "what this run built" (#4290).
       published_digest: ${{ steps.reuse.outputs.published_digest }}
+      # Every tag this commit should end up under (channel, plus :c-<sha> on
+      # main). `promote` writes them from `digest` once the smoke has passed.
+      tags: ${{ steps.tag.outputs.tags }}
     # A plain `needs` — no `always()`. That expression existed to let a
     # SKIPPED sign-extension through on a tag push while still blocking a
     # FAILED one. With no tag trigger, sign-extension always runs, so the
     # default behaviour is exactly what we want: a failed sign skips build-web
     # rather than shipping an image without its XPI.
     needs: [sign-extension, lint, extension-version, backend-lint-and-test,
-            frontend-build, integration]
+            frontend-build, extension-test, integration]
     # The lanes in that list are THE GATE (2026-09-23) — see sign-extension's
     # copy of this comment for why, including the property that a SKIPPED lane
     # blocks as firmly as a failing one. They are repeated here rather than
@@ -871,8 +998,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           # Not the triggering ref — see the `env:` block at the top. On a
-          # scheduled refresh this is `main`; on everything else it is the ref
-          # that fired, so this is a no-op on every ordinary path.
+          # scheduled refresh this is `main`; on everything else it is the
+          # commit that fired, the same one the lanes above tested.
           ref: ${{ env.BUILD_REF }}
           # Full history: this job RE-DERIVES the extension version rather than
           # being handed it, and a depth-1 clone derives a wrong, too-low value
@@ -1108,11 +1235,13 @@ jobs:
           # WHERE THE BUILD PUBLISHES, which is not always the channel — and
           # whether the channel then has to be written separately.
           #
-          # On a push the build writes the channel tag directly: the bytes came
-          # from a commit, and a commit is the thing CI tests. Nothing to hold
-          # it behind.
+          # Every build writes a CANDIDATE tag, never the channel (#4310):
+          # :dev-candidate / :latest-candidate on a push, :refresh-candidate on
+          # the refresh. `promote` moves the channel only after smoke-web has
+          # booted the bytes. The lanes prove the SOURCE; only the smoke proves
+          # the BYTES, and the two are not the same claim.
           #
-          # On the scheduled refresh it writes a CANDIDATE tag instead. A
+          # The refresh is the case that made this obvious. A
           # refresh rebuilds against freshly resolved base images, and the web
           # image's runtime is a line of UNPINNED Debian packages (ffmpeg,
           # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
@@ -1142,7 +1271,10 @@ jobs:
             echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
             echo "promote=true" >> "$GITHUB_OUTPUT"
           else
-            echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
+            # A push builds to a per-channel candidate too (#4310). The channel
+            # tag moves in `promote`, after smoke-web has booted these bytes —
+            # so a broken image never reaches the tag deployments follow.
+            echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
             echo "promote=false" >> "$GITHUB_OUTPUT"
           fi
 
@@ -1318,8 +1450,11 @@ jobs:
           # `no-cache: true`, which is a much larger version of the same
           # churn #3265 is about.
           #
-          # Only on the schedule. An ordinary push wants the cached base.
-          pull: ${{ env.IS_REFRESH == 'true' }}
+          # Only on the refresh and on a forced build. An ordinary push wants
+          # the cached base. A forced build pulls because forcing is how the
+          # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
+          # keeps the old base layers would be the same image again.
+          pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
           # ONE tag, the channel's. Every other tag is written by the step
           # below, registry-side. buildx here pushes the first tag to the
           # registry and then re-pushes the rest through the DOCKER driver,
@@ -1418,6 +1553,16 @@ jobs:
           TAGS: ${{ steps.tag.outputs.tags }}
         run: |
           set -euf
+          # A BUILD publishes nothing from here (#4310). Its bytes sit on the
+          # candidate tag until smoke-web has booted them; `promote` then
+          # writes the channel tag AND :c-<sha> from this run's digest. Writing
+          # :c-<sha> here would publish an immutable rollback tag for bytes
+          # that might then fail the smoke.
+          if [ -n "${BUILT_DIGEST:-}" ]; then
+            echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
+            echo "repoint: after the smoke; nothing to write here."
+            exit 0
+          fi
           # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
           #
           # This step used to copy from the channel tag by NAME. Nothing
@@ -1511,14 +1656,9 @@ jobs:
   # source tree, and not a static inspection: `ffmpeg -version` exiting 0 would
   # pass while a codec removal broke every thumbnail in the library.
   #
-  # Refresh-only. On a push the bytes came from a commit, and a commit is what
-  # the lanes above already test — and since 2026-09-23 they gate the build, so
-  # those bytes could not exist without them having passed.
-  #
-  # Reports a verdict; it does not yet gate the promote (milestone 362 step 4).
-  # Landing the gate and the thing it gates in one change would mean the first
-  # time anyone saw this job run would also be the first time it could stop a
-  # publish.
+  # It gates `promote` on every trigger (#4310). The lanes above prove the
+  # source; they cannot see the bytes, and a Dockerfile or base change breaks
+  # the bytes without touching the source.
   smoke-web:
     needs: [build-web]
     # Every run that actually BUILT something, not just the weekly refresh.
@@ -1530,13 +1670,10 @@ jobs:
     # and were smoked when they were built. Re-smoking them would burn two
     # minutes to re-learn a fact.
     #
-    # HONEST LIMIT, and it is the reason #4299 exists: on a push this runs
-    # AFTER build-web has written the channel tag, so it detects rather than
-    # gates. Rule 164's verify_with asks for the check BETWEEN build and push.
-    # Closing that needs the push path to adopt the candidate-then-promote
-    # shape the refresh already has — per-channel candidate tags, promote
-    # learning its channel, and the :c-<sha> repoint moving after the gate.
-    # That is a redesign of the production publish path and is its own task.
+    # This GATES the publish on every trigger (#4310): builds land on a
+    # candidate tag, and `promote` needs this job, so :dev / :latest and a
+    # built :c-<sha> move only after it passes. Rule 164's verify_with — the
+    # check BETWEEN build and push.
     #
     # NO `if:` — this job always runs (#4323). It used to be gated on the
     # build having published something, which skipped it on a reuse hit. That
@@ -1583,7 +1720,7 @@ jobs:
           --health-timeout 5s
           --health-retries 10
       redis:
-        image: redis:7-alpine
+        image: redis:8-alpine
         options: >-
           --health-cmd "redis-cli ping"
           --health-interval 10s
@@ -1608,9 +1745,20 @@ jobs:
           # Service discovery mirrors the integration lane above: these jobs run
           # in a container against a mounted docker socket, so the services are
           # SIBLINGS reachable by IP, not by hostname.
-          PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1)
-          test -n "$PG" && test -n "$RD"
+          # Only THIS job's services. The runner's docker daemon is shared, so another
+          # repo's job can be running beside this one, and a job-name filter matches its
+          # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
+          # Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
+          # <n> from our own container (its id is in the /etc/hostname bind mount) and
+          # require exactly one match.
+          SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
+          SELF=${SELF:-$(hostname)}
+          TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
+          test -n "$TASK"
+          PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
+          test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
+          RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
+          test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
           test -n "$PG_IP" && test -n "$RD_IP"
@@ -1954,88 +2102,228 @@ jobs:
   # smoked it, and it should not be read as if it were.
   promote:
     needs: [build-web, build-agent, smoke-web]
-    # Only a refresh publishes through a candidate; a push writes its channel
-    # tag directly from the build. Reads the same reuse-step decision the build
-    # took, via a job output — a job's `if:` cannot see the `env` context.
-    if: needs.build-web.outputs.candidate == 'true'
+    # THE PUBLISH (#4310). Both image jobs build to a candidate tag — a push to
+    # :dev-candidate / :latest-candidate, the weekly refresh to
+    # :refresh-candidate — and nothing names those bytes under a tag anybody
+    # deploys until this job runs. It runs only when every job in `needs`
+    # SUCCEEDED (a job-level `if:` without a status function implies
+    # success()), so a failed smoke — or a skipped one, which is not the same
+    # as a passing one (run 5290) — leaves :dev / :latest on the last build
+    # that worked. Rule 164's verify_with: the check sits BETWEEN build and
+    # push.
+    #
+    # Per image, from the DIGEST that image's job built (#4290): a tag can
+    # move between the build and this job, a digest cannot. An image whose job
+    # hit reuse built nothing and has nothing to promote — its build job
+    # already wrote its :c-<sha> from the channel tag, which "hit" proved
+    # carries this commit.
+    if: github.event_name != 'pull_request'
     runs-on: python-ci
     container:
       image: git.fabledsword.com/bvandeusen/ci-python:3.14
     steps:
-      - name: Point the channel tags at the smoked candidates
+      - name: Point this commit's tags at the smoked builds
         env:
           TOKEN: ${{ secrets.RELEASE_TOKEN }}
           ACTOR: ${{ github.actor }}
+          WEB_DIGEST: ${{ needs.build-web.outputs.digest }}
+          WEB_TAGS: ${{ needs.build-web.outputs.tags }}
+          AGENT_DIGEST: ${{ needs.build-agent.outputs.digest }}
+          AGENT_TAGS: ${{ needs.build-agent.outputs.tags }}
         run: |
           set -eu
-          # `latest` is not a guess: a refresh always builds `main` (BUILD_REF),
-          # and the "must have checked out main" guard in every build job fails
-          # the run if that did not hold. So the channel is main's.
-          TAG=latest
           FAILED=""
+          MOVED=0
 
-          for NAME in fabledcurator fabledcurator-agent; do
+          promote() {
+            NAME="$1"; DIGEST="$2"; TAGS="$3"
             REPO="bvandeusen/$NAME"
-            echo "promote: $REPO"
+            if [ -z "$DIGEST" ]; then
+              echo "promote: $NAME — no build this run (reuse hit); nothing to publish"
+              return 0
+            fi
+            echo "promote: $NAME $DIGEST"
 
             # Registry auth is its own token exchange — `docker login`
             # authenticates the docker client, not curl. Deadline on every call
             # (rule 156): a registry that stops answering must fail this step,
-            # not hang the weekly refresh until the job times out.
+            # not hang the run until the job times out.
             BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
               "https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
               | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
 
             # Ask for the IMAGE manifest media types only. Offering the index
-            # types too would let the registry hand back an index if one ever
-            # existed at this tag, and we would faithfully copy the thing this
-            # whole approach exists to avoid creating.
+            # types too would let the registry hand back an index, and we would
+            # faithfully copy the thing this approach exists to avoid creating.
             ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
             CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
                    -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
-                   "https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
+                   "https://git.fabledsword.com/v2/$REPO/manifests/$DIGEST" \
                  && tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
             test -n "$CT"
-            SRC=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
-            echo "promote:   candidate $SRC ($CT)"
 
             # NOT `imagetools create`. That wraps its source in an INDEX, and
             # `.Image.Config.Labels` does not resolve through one — the
             # fc.revision the reuse check reads off the channel tag would come
             # back empty, every later push would miss and rebuild, and nothing
             # would go red (#3183, run 4751). A manifest PUT is what "make this
-            # tag name that image" means at the registry: same bytes, same media
-            # type, same digest, no layer transfer.
-            curl -fsS --max-time 120 -X PUT \
-              -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
-              --data-binary @manifest.json \
-              "https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
+            # tag name that image" means at the registry: same bytes, same
+            # media type, same digest, no layer transfer. (It also makes a
+            # built :c-<sha> a plain image rather than the index the old
+            # repoint left.)
+            IFS=,
+            for REF in $TAGS; do
+              TAG="${REF##*:}"
+              curl -fsS --max-time 120 -X PUT \
+                -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
+                --data-binary @manifest.json \
+                "https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
 
-            # Read it back. A PUT that returned 2xx but landed something else is
-            # exactly the silent-and-plausible failure this pipeline keeps
-            # producing, and the check costs one request.
-            NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
-                    -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
-                    "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
-                  | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
-            if [ "$NOW" != "$SRC" ]; then
-              echo "promote:   FAILED — $NAME:$TAG is $NOW, expected $SRC" >&2
-              FAILED="$FAILED $NAME"
-              continue
-            fi
-            echo "promote:   $NAME:$TAG now names $NOW"
-          done
+              # Read it back. A PUT that returned 2xx but landed something else
+              # is exactly the silent-and-plausible failure this pipeline keeps
+              # producing, and the check costs one request.
+              NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
+                      -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
+                      "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
+                    | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
+              if [ "$NOW" != "$DIGEST" ]; then
+                echo "promote:   FAILED — $NAME:$TAG is $NOW, expected $DIGEST" >&2
+                FAILED="$FAILED $NAME:$TAG"
+                continue
+              fi
+              echo "promote:   $NAME:$TAG now names $NOW"
+              MOVED=$((MOVED + 1))
+            done
+            unset IFS
+          }
+
+          promote fabledcurator "$WEB_DIGEST" "$WEB_TAGS"
+          promote fabledcurator-agent "$AGENT_DIGEST" "$AGENT_TAGS"
 
           if [ -n "$FAILED" ]; then
             echo "" >&2
             echo "promote: FAILED for:$FAILED" >&2
-            echo "promote: the channel tags are now INCONSISTENT — some images" >&2
-            echo "promote: moved and some did not. Re-run this refresh; the" >&2
-            echo "promote: candidates are still published and the promote is" >&2
-            echo "promote: idempotent." >&2
+            echo "promote: the tags are now INCONSISTENT — some moved and some" >&2
+            echo "promote: did not. Re-run this workflow; the builds are still" >&2
+            echo "promote: published by digest and the promote is idempotent." >&2
             exit 1
           fi
-          echo "promote: both channel tags moved"
+          echo "promote: $MOVED tag(s) written"
+
+  # The refresh rebuilds `:dev` as well as `:latest`.
+  #
+  # The refresh run itself builds main and publishes `:latest`. `:dev` needs a
+  # build of dev's own source with FC_CHANNEL=dev baked in, so copying main's
+  # refreshed bytes onto `:dev` is not the same thing: the image would report
+  # itself as main, and the reuse check would then keep it that way. Instead
+  # this job dispatches an ordinary dev run with `force_build`. That run goes
+  # through the lanes, the smoke and `promote` like any push, and a forced
+  # build pulls fresh bases (see `pull:` in both build jobs).
+  #
+  # Without this, an artifact whose source is quiet on dev (the agent, most
+  # weeks) keeps the base layers of its last build indefinitely, because
+  # reuse skips its build on every push.
+  #
+  # The dispatched run shares this run's concurrency group (both are on the
+  # dev ref), so it queues until this run has finished. It runs whatever
+  # `promote` did: a failed `:latest` refresh is no reason to leave `:dev`
+  # stale, and `unattended-failure` reports the failure either way.
+  refresh-dev:
+    needs: [promote]
+    if: >-
+      !cancelled() && (github.event_name == 'schedule'
+      || format('{0}', github.event.inputs.refresh) == 'true')
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    steps:
+      - name: Dispatch a forced dev build
+        env:
+          TOKEN: ${{ secrets.RELEASE_TOKEN }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+        run: |
+          set -eu
+          CODE=$(curl -sS --max-time 30 -o response.txt -w '%{http_code}' -X POST \
+            -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
+            -d '{"ref":"dev","inputs":{"force_build":"true"}}' \
+            "$API/actions/workflows/build.yml/dispatches")
+          echo "refresh-dev: dispatch answered HTTP $CODE"
+          case "$CODE" in
+            2??) echo "refresh-dev: a forced dev run is queued behind this one" ;;
+            *)
+              cat response.txt >&2
+              echo "refresh-dev: the dispatch was refused. A 403 means RELEASE_TOKEN" >&2
+              echo "refresh-dev: lacks write:repository (see the scope list at the top)." >&2
+              exit 1 ;;
+          esac
+
+  # A run nobody is watching reports its own failure.
+  #
+  # A push run is watched by whoever pushed it. The weekly refresh and the dev
+  # run it dispatches are not, and a red run on the Actions page alone goes
+  # unseen. So any failed run that is not a push or a pull request opens an
+  # issue, or comments on the one already open, naming the run and the jobs
+  # that failed. A manual dispatch that fails reports here too; closing the
+  # issue once it is dealt with is what keeps the next failure visible.
+  #
+  # The issue is written with RELEASE_TOKEN, so it is authored by the token's
+  # owner, and the forge does not notify people of their own actions. It
+  # shows in the repository's open issues rather than as an email.
+  unattended-failure:
+    needs: [lint, extension-version, backend-lint-and-test, frontend-build,
+            extension-test, integration, sign-extension, build-web, smoke-web,
+            build-agent, promote, refresh-dev]
+    if: >-
+      failure() && github.event_name != 'push'
+      && github.event_name != 'pull_request'
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    steps:
+      - name: Open or update the failure issue
+        env:
+          TOKEN: ${{ secrets.RELEASE_TOKEN }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          EVENT: ${{ github.event_name }}
+          REF: ${{ github.ref_name }}
+          NEEDS: ${{ toJSON(needs) }}
+        run: |
+          python3 - <<'EOF'
+          import json, os, urllib.parse, urllib.request
+
+          TITLE = "CI: an unattended build.yml run failed"
+          api, token = os.environ["API"], os.environ["TOKEN"]
+
+          def call(method, path, body=None):
+              req = urllib.request.Request(
+                  api + path, method=method,
+                  data=json.dumps(body).encode() if body is not None else None,
+                  headers={"Authorization": f"token {token}",
+                           "Content-Type": "application/json"})
+              with urllib.request.urlopen(req, timeout=30) as resp:
+                  return json.load(resp)
+
+          needs = json.loads(os.environ["NEEDS"])
+          failed = sorted(j for j, v in needs.items() if v.get("result") == "failure")
+          text = (f"Run: {os.environ['RUN_URL']}\n"
+                  f"Trigger: `{os.environ['EVENT']}` on `{os.environ['REF']}`\n"
+                  f"Failed jobs: {', '.join(f'`{j}`' for j in failed) or '(none reported)'}\n")
+
+          q = urllib.parse.urlencode({"state": "open", "type": "issues", "q": TITLE})
+          existing = [i for i in call("GET", f"/issues?{q}") if i["title"] == TITLE]
+          if existing:
+              n = existing[0]["number"]
+              call("POST", f"/issues/{n}/comments", {"body": text})
+              print(f"unattended-failure: commented on open issue #{n}")
+          else:
+              body = ("A build.yml run that nobody watches failed. Read the run's "
+                      "failing job logs; close this issue once it is fixed, so the "
+                      "next failure opens a new one.\n\n" + text)
+              n = call("POST", "/issues", {"title": TITLE, "body": body})["number"]
+              print(f"unattended-failure: opened issue #{n}")
+          EOF
+
   build-agent:
     # THE GATE (2026-09-23). Every lane above must have PASSED before this job
     # exists at all — so a red suite does not produce an image, let alone push
@@ -2048,7 +2336,14 @@ jobs:
     # itself through a job-level `if:` that could not read `env`, and a design
     # where only a FAILED gate blocks would have published unverified images
     # while reporting success.
-    needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
+    needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
+            integration]
+    # What `promote` needs to publish this image once the smoke has passed:
+    # the manifest this run built (empty on a reuse hit) and every tag it
+    # belongs under. Same meaning as build-web's outputs of the same names.
+    outputs:
+      digest: ${{ steps.build.outputs.digest }}
+      tags: ${{ steps.tag.outputs.tags }}
     # A pull_request run is the lanes and nothing else. This is the ONLY thing
     # separating "validate a Renovate bump" from "publish a Renovate bump", so
     # it is stated on each publishing job rather than inferred from a `needs`
@@ -2061,8 +2356,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           # Not the triggering ref — see the `env:` block at the top. On a
-          # scheduled refresh this is `main`; on everything else it is the ref
-          # that fired, so this is a no-op on every ordinary path.
+          # scheduled refresh this is `main`; on everything else it is the
+          # commit that fired, the same one the lanes above tested.
           ref: ${{ env.BUILD_REF }}
           # Full history: this job derives its artifact's version from the
           # commit its shipped files last changed in (milestone 313). A
@@ -2240,11 +2535,13 @@ jobs:
           # WHERE THE BUILD PUBLISHES, which is not always the channel — and
           # whether the channel then has to be written separately.
           #
-          # On a push the build writes the channel tag directly: the bytes came
-          # from a commit, and a commit is the thing CI tests. Nothing to hold
-          # it behind.
+          # Every build writes a CANDIDATE tag, never the channel (#4310):
+          # :dev-candidate / :latest-candidate on a push, :refresh-candidate on
+          # the refresh. `promote` moves the channel only after smoke-web has
+          # booted the bytes. The lanes prove the SOURCE; only the smoke proves
+          # the BYTES, and the two are not the same claim.
           #
-          # On the scheduled refresh it writes a CANDIDATE tag instead. A
+          # The refresh is the case that made this obvious. A
           # refresh rebuilds against freshly resolved base images, and the web
           # image's runtime is a line of UNPINNED Debian packages (ffmpeg,
           # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
@@ -2273,7 +2570,10 @@ jobs:
           if [ "${IS_REFRESH:-}" = "true" ]; then
             echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
           else
-            echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
+            # A push builds to a per-channel candidate too (#4310). The channel
+            # tag moves in `promote`, after smoke-web has booted these bytes —
+            # so a broken image never reaches the tag deployments follow.
+            echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
           fi
 
           # Compare VALUES, never exit codes. Measured on buildx v0.36.1
@@ -2362,8 +2662,11 @@ jobs:
           # `no-cache: true`, which is a much larger version of the same
           # churn #3265 is about.
           #
-          # Only on the schedule. An ordinary push wants the cached base.
-          pull: ${{ env.IS_REFRESH == 'true' }}
+          # Only on the refresh and on a forced build. An ordinary push wants
+          # the cached base. A forced build pulls because forcing is how the
+          # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
+          # keeps the old base layers would be the same image again.
+          pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
           # ONE tag, the channel's. Every other tag is written by the step
           # below, registry-side. buildx here pushes the first tag to the
           # registry and then re-pushes the rest through the DOCKER driver,
@@ -2464,6 +2767,16 @@ jobs:
           TAGS: ${{ steps.tag.outputs.tags }}
         run: |
           set -euf
+          # A BUILD publishes nothing from here (#4310). Its bytes sit on the
+          # candidate tag until smoke-web has booted them; `promote` then
+          # writes the channel tag AND :c-<sha> from this run's digest. Writing
+          # :c-<sha> here would publish an immutable rollback tag for bytes
+          # that might then fail the smoke.
+          if [ -n "${BUILT_DIGEST:-}" ]; then
+            echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
+            echo "repoint: after the smoke; nothing to write here."
+            exit 0
+          fi
           # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
           #
           # This step used to copy from the channel tag by NAME. Nothing

.forgejo/workflows/extension.yml

@@ -1,86 +0,0 @@
-name: extension
-# Lint + unit tests. Deliberately NOT a publishing lane, which is why it is
-# not part of build.yml's gate. The sign-and-publish dance moved into build.yml's
-# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
-# because sign-extension runs as a build-web dependency in the SAME workflow,
-# eliminating the prior race between build.yml and a separate extension.yml.
-# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
-on:
-  push:
-    branches: [dev, main]
-    paths:
-      - 'extension/**'
-      - '.forgejo/workflows/extension.yml'
-      # test/version.spec.js asserts things ABOUT build.yml — that it does not
-      # inline the packaged-file set, and that build.yml derives
-      # the shipped version rather than reading it out of the repo. A
-      # workflow-only edit can therefore break this suite, so it has to trigger
-      # it. build.yml joined the list at milestone 271 step 5, when the spec
-      # started asserting against it.
-      - '.forgejo/workflows/build.yml'
-  pull_request:
-    branches: [main]
-    paths:
-      - 'extension/**'
-      - '.forgejo/workflows/build.yml'
-  workflow_dispatch:
-
-jobs:
-  lint:
-    runs-on: python-ci
-    container:
-      image: node:24-bookworm-slim
-    steps:
-      - uses: actions/checkout@v4
-      # Not --no-save: vitest and web-ext are both real devDependencies now,
-      # and the suite needs vitest resolvable from node_modules.
-      - name: Install dev dependencies
-        run: cd extension && npm install --no-audit --no-fund
-      - name: Lint
-        run: cd extension && npm run lint
-      # Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
-      # package version-consistency checks. No browser, no network.
-      - name: Unit tests
-        run: cd extension && npm run test:unit
-
-      # Everything else about packaging is asserted against our own declaration
-      # of what ships. This is the only check that asks web-ext what it ACTUALLY
-      # put in the archive. Until now that was an unverified assumption about
-      # glob semantics — and a fragile one: `test/**` reaches web-ext intact
-      # only because callers `set -f` first, so losing that quoting would
-      # silently start shipping dev files with no other signal.
-      - name: Verify XPI contents
-        run: |
-          set -eu
-          command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
-          cd extension
-          npm run build
-          ZIP=$(ls web-ext-artifacts/*.zip | head -1)
-          echo "=== packaged entries in $ZIP ==="
-          unzip -Z1 "$ZIP" | sort
-          echo "=== end ==="
-          ENTRIES=$(unzip -Z1 "$ZIP")
-          fail=0
-          # Must NOT ship: repo infrastructure with no business in a user's browser.
-          for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
-            if echo "$ENTRIES" | grep -q "^$pat"; then
-              echo "ERROR: '$pat' was packaged into the XPI but must not be"
-              fail=1
-            fi
-          done
-          # Must ship: if an exclusion pattern ever over-matches, the extension
-          # breaks at runtime rather than at build time, so assert presence too.
-          for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js'; do
-            if ! echo "$ENTRIES" | grep -q "^$req$"; then
-              echo "ERROR: '$req' is missing from the XPI"
-              fail=1
-            fi
-          done
-          for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
-            if ! echo "$ENTRIES" | grep -q "^$dir"; then
-              echo "ERROR: nothing from '$dir' was packaged"
-              fail=1
-            fi
-          done
-          [ "$fail" -eq 0 ] || exit 1
-          echo "XPI contents verified."

Dockerfile

@@ -36,11 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libwebp7 \
         libpng16-16 \
         ca-certificates \
-    # opencv-python-headless (via requirements-ml.txt) links these even in its
-    # headless build. Came from Dockerfile.ml when the images merged
-    # (milestone 422 step 6).
-        libgl1 \
-        libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
@@ -61,7 +56,8 @@ RUN pip install -r requirements.txt
 #
 #   torch 2.12.1+cpu wheel     192.3 MB
 #   torchvision 0.27.1+cpu       1.8 MB
-#   transformers / onnxruntime / opencv / sklearn and friends
+#   transformers / onnxruntime / opencv / sklearn and friends (opencv and
+#   onnxruntime since dropped, #1451 — nothing here imported them)
 #                               62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB
 #   largest newly-pushed layer 222.07 MB
 #
@@ -85,7 +81,7 @@ RUN pip install -r requirements.txt
 # CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the
 # GPU agent is a separate service with its own image.
 RUN pip install --index-url https://download.pytorch.org/whl/cpu \
-        "torch>=2.12,<3.0" "torchvision>=0.27,<0.28"
+        "torch>=2.14" "torchvision>=0.29"
 RUN pip install -r requirements-ml.txt
 
 # Where the model lands. Deliberately NOT a VOLUME instruction: that mints an

README.md

@@ -22,7 +22,8 @@ through afterwards.
 - **ML tagging.** Runs image models in-container to suggest tags, group
   characters, find near-duplicates and power similarity search. Suggestions are
   reviewable — it proposes, you confirm, and it learns which proposals you keep
-  rejecting.
+  rejecting. It ships switched off: turn it on under Settings → System when
+  you want it, and it fetches its model weights then.
 - **Deduplication and provenance.** Everything that arrives is hashed and
   deduplicated by content, metadata sidecars are read wherever the source
   writes them, and every file keeps a record of where it came from.
@@ -113,15 +114,19 @@ that cannot decrypt anything, leaving an instance that looks healthy while every
 paywalled download fails. Making you say so once, on an empty install, is the
 price of that not happening silently later.
 
-**Which means: back up `./images/secrets/` alongside your database.** It is the
-only thing that can read your stored credentials. A database restored without it
-needs every credential entered again by hand.
+**Your account logins are sealed with that key, and no backup carries either.**
+FabledCurator's backups (below) leave the key and the logins out on purpose, so
+a stolen backup holds no way into your accounts. After a restore, sign in to
+each account again through the browser extension.
 
 A few other things are worth knowing about the first few minutes:
 
-- **The ML worker downloads its model weights on first boot**, several GB from
-  HuggingFace into `./models`. Until that finishes, tagging is queued rather
-  than broken. It is idempotent — a restart resumes rather than refetches.
+- **ML tagging starts switched off, and nothing is downloaded at boot.** Give
+  the ML lane a slot under **Settings → System** and it fetches its model
+  weights then — several GB from HuggingFace into `./models`, shown as a job
+  under **Settings → Activity** that you can watch and retry. Until it
+  finishes, tagging is queued rather than broken, and the fetch only takes
+  what is missing, so turning the lane off and on again does not refetch.
 - **The gallery starts empty**, and that is the expected state. Add a creator
   under **Subscriptions** and it fills as posts come down.
 - **If you already have a library on disk**, there is no screen that imports
@@ -173,6 +178,52 @@ Run it for a burst of tagging, stop it to get your card back. It deploys from
 `agent/docker-compose.yml`, not the main stack — see
 [agent/README.md](agent/README.md).
 
+## Backing up and restoring
+
+Backups live under **Settings → Maintenance → Backups**. With nothing else set
+up, the database is dumped nightly into `./images/_backups`, which is on the
+same disk as what it protects. Images and attachments are backed up only to an
+off-site destination.
+
+To keep a copy off the machine, set up an off-site destination on the same
+screen: any S3 bucket (Fabled Archives, MinIO, AWS) and a key that may add and
+read objects. Each night FabledCurator then sends the database as a dated dump
+under `database/`, and every image and attachment once, named by its
+fingerprint, under `media/`. It never deletes from the bucket; the bucket's own
+retention thins old dumps. The first copy of a large library takes hours; a
+restart along the way carries on where it stopped rather than starting over.
+
+Once a week the bucket is checked against what was sent, and anything missing
+goes again in the next run. When the newest database backup is more than a day
+old or failed, or the check found files missing, the **Backups** row on
+**Settings → System** and the dot beside the brand turn amber and say why.
+
+**Test a backup** proves a dump restores without touching your install: the dump
+is loaded into a separate, temporary database on the same Postgres server,
+every table is checked against the row counts recorded as the dump was taken,
+the copy is brought up to this version, and then it is deleted. It needs about
+as much free space on the database's disk as the database itself, while it
+runs. A dump taken before counts were recorded is shown beside the live
+database instead.
+
+To rebuild after losing the server, start a fresh install (first boot as
+above), enter the same destination, and under **Restore from the bucket** choose
+a dump. The database is replaced, then every file is fetched back to its place
+and checked against its fingerprint as it arrives; progress shows on the same
+screen. **Fetch missing files** does only the second half, for a library that
+lost files but kept its database.
+
+If the web page isn't reachable, the same restore runs from a shell:
+
+```bash
+docker compose exec web python -m backend.app.scripts.restore list
+docker compose exec web python -m backend.app.scripts.restore database --dump latest --yes
+```
+
+On an install with no destination saved yet, add `--endpoint`, `--bucket` and
+`--access-key`, and put the secret key in `FC_BACKUP_SECRET_KEY` (or type it at
+the prompt).
+
 ## Upgrading
 
 ```bash
@@ -271,11 +322,12 @@ Four deployable pieces, built by `.forgejo/workflows/build.yml`:
 
 ## CI / Forgejo setup
 
-Three workflows: `build.yml` (the five verification lanes — lint,
-extension-version check, backend unit tests, frontend build, integration — and
-then sign + publish), `extension.yml` (extension lint, vitest, XPI content
-verification), and `release.yml`, which runs only on a `v*` tag and publishes a
-changelog without building anything.
+Two workflows that matter here: `build.yml` (the six verification lanes — lint,
+extension-version check, backend unit tests, frontend build, extension lint +
+vitest + XPI content check, integration — and then sign + publish), and
+`release.yml`, which runs only on a `v*` tag and publishes a changelog without
+building anything. The extension lane was its own `extension.yml` until
+milestone 429, which let a red extension suite sign and ship the XPI anyway.
 
 **The lanes and the publish are one workflow on purpose.** They were two
 (`ci.yml` and `build.yml`) until 2026-09-23, on the same push trigger, which

agent/Dockerfile

@@ -1,10 +1,21 @@
 # FabledCurator GPU agent — runs on the desktop with the GPU.
-# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs
-# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open
-# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
-# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are
-# built against (CUDA 13 has only nascent ONNX Runtime support).
-FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04
+#
+# The `base` flavour, not `cudnn-runtime`: CUDA and cuDNN arrive as the
+# `nvidia-*` pip packages torch and onnxruntime-gpu depend on, so the base only
+# has to hand the container the driver (it sets NVIDIA_VISIBLE_DEVICES /
+# NVIDIA_DRIVER_CAPABILITIES for the Container Toolkit). Until #1451 this was
+# `12.9.2-cudnn-runtime` under a `torch==2.6.0+cu124` — and requirements.txt then
+# REPLACED that torch with PyPI's CUDA-13 build (ultralytics pulls torchvision,
+# which pulls its matching torch), beside a CUDA-13 onnxruntime-gpu. The image
+# ran CUDA 13 on a CUDA-12 base, carrying ~3 GB of base libraries and a ~3 GB
+# torch nothing loaded: 10 GB compressed.
+#
+# 13.0 because that is the line both wheels are built for (torch's cu130 index,
+# onnxruntime-gpu's `nvidia-cuda-runtime~=13.0`). Needs an NVIDIA driver that
+# supports CUDA 13 (580+); fc_agent/accel.py logs at startup whether torch and
+# onnxruntime actually got the GPU, since both fall back to the CPU silently.
+# ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
+FROM nvidia/cuda:13.0.3-base-ubuntu24.04
 
 # PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
 # managed (PEP 668), so a global `pip install` errors without this. It's a
@@ -16,10 +27,12 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
-# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN
-# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first
-# + separately so the GPU build of torch is deterministic and layer-cached.
-RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
+# torch AND torchvision from the cu130 index, together and first. Installing
+# torch alone is what let the next step swap it out: ultralytics needs
+# torchvision, PyPI's torchvision pins its own torch, and pip replaced ours to
+# match. With both present, requirements.txt finds them satisfied.
+RUN pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu130 \
+        torch torchvision
 COPY requirements.txt .
 RUN pip3 install --no-cache-dir -r requirements.txt
 COPY fc_agent ./fc_agent

agent/README.md

@@ -15,7 +15,22 @@ sudo pacman -S nvidia-container-toolkit
 sudo nvidia-ctk runtime configure --runtime=docker
 sudo systemctl restart docker
 # verify:
-docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi
+docker run --rm --gpus all nvidia/cuda:13.0.3-base-ubuntu24.04 nvidia-smi
+# the header's CUDA version must be 13.0 or later (driver 580+)
+```
+
+### After a driver update: regenerate the CDI spec
+If the agent's first log lines say `accel: torch is NOT on the GPU` or report
+`cudaGetDeviceCount: unknown error (999)` while `nvidia-smi` still works, the
+toolkit's saved device list (`/etc/cdi/nvidia.yaml`) is out of date. The
+`nvidia-uvm` device number changes between driver versions, and a spec
+generated before the update hands the container a device node that no longer
+exists (2026-09-24: host `511,0`, container `235,0`). Compare
+`ls -l /dev/nvidia-uvm` on the host with the same inside the container, then:
+```sh
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+# if your toolkit ships it, this keeps it current on every driver update:
+sudo systemctl enable --now nvidia-cdi-refresh.path
 ```
 
 ## 1. Get a token

agent/fc_agent/accel.py

@@ -0,0 +1,124 @@
+"""Which accelerator each runtime actually got — reported once, at startup.
+
+The agent has two GPU runtimes and both fall back to the CPU without raising:
+torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils
+detector + CCIP models) when its CUDA provider cannot load its libraries. A
+fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the
+image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base
+(#1451), and whether the ONNX half was on the GPU could not be answered from
+anything the agent had ever logged.
+
+Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA
+provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image
+they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT
+1.21+) loads them from there, so the provider resolves them by soname.
+
+Stdlib-only at import, so the unit suite can import it — torch and onnxruntime
+are imported inside the functions.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import importlib
+import logging
+from pathlib import Path
+
+log = logging.getLogger("fc_agent.accel")
+
+# Filled by report(); /status carries it so the page can show it too.
+LAST: dict = {}
+
+
+def torch_status(imp=importlib.import_module) -> dict:
+    try:
+        torch = imp("torch")
+    except Exception as e:
+        return {"device": "unavailable", "error": str(e)}
+    out = {"version": torch.__version__, "cuda_build": torch.version.cuda}
+    if torch.cuda.is_available():
+        out["device"] = "cuda"
+        out["gpu"] = torch.cuda.get_device_name(0)
+    else:
+        out["device"] = "cpu"
+    return out
+
+
+def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict:
+    try:
+        ort = imp("onnxruntime")
+    except Exception as e:
+        return {"device": "unavailable", "error": str(e)}
+    out = {"version": ort.__version__, "providers": list(ort.get_available_providers())}
+    if "CUDAExecutionProvider" not in out["providers"]:
+        out["device"] = "cpu"
+        return out
+    preload = getattr(ort, "preload_dlls", None)
+    if preload is not None:
+        try:
+            preload()
+        except Exception as e:
+            out["preload_error"] = str(e)
+    # "Available" only means the build HAS the provider. Loading its library is
+    # what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are
+    # missing, and the one a session would otherwise fail silently on.
+    capi = Path(ort.__file__).parent / "capi"
+    try:
+        load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL)
+        load(str(capi / "libonnxruntime_providers_cuda.so"))
+    except OSError as e:
+        out["device"] = "cpu"
+        out["error"] = str(e)
+        return out
+    # Loading proves the libraries resolve, NOT that a GPU can be used: on
+    # 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with
+    # "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA
+    # runtime for a device initialises the driver the provider would use.
+    error = _cuda_device_error(load)
+    out["device"] = "cpu" if error else "cuda"
+    if error:
+        out["error"] = error
+    return out
+
+
+def _cuda_device_error(load=ctypes.CDLL) -> str | None:
+    """None when the CUDA runtime can reach a device, else why it cannot."""
+    try:
+        cudart = load("libcudart.so.13")
+    except OSError as e:
+        return str(e)
+    count = ctypes.c_int(0)
+    rc = cudart.cudaGetDeviceCount(ctypes.byref(count))
+    if rc != 0:
+        cudart.cudaGetErrorString.restype = ctypes.c_char_p
+        return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})"
+    return None if count.value > 0 else "no CUDA device visible"
+
+
+def summary() -> dict | None:
+    """The report as FabledCurator stores it: each runtime's device, and why
+    when it is not the GPU. Sent on every lease and heartbeat, so the System
+    view can call a running agent that fell back to the CPU "degraded" rather
+    than "running" — the 2026-09-24 fallback went unseen for weeks because
+    only this agent's own log said so. None before report() has run."""
+    if not LAST:
+        return None
+    out = {}
+    for name, s in LAST.items():
+        entry = {"device": s.get("device")}
+        if s.get("error"):
+            entry["error"] = str(s["error"])[:200]
+        out[name] = entry
+    return out
+
+
+def report() -> dict:
+    """Check both runtimes, log the result, and keep it for /status."""
+    LAST.clear()
+    LAST.update(torch=torch_status(), onnx=onnx_status())
+    for name, s in LAST.items():
+        if s.get("device") == "cuda":
+            log.info("accel: %s on GPU (%s)", name, s)
+        else:
+            log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s)
+    return dict(LAST)

agent/fc_agent/app.py

@@ -11,7 +11,7 @@ import logging
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, JSONResponse
 
-from . import logbuf
+from . import accel, logbuf
 from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version
 from .config import Config
 from .gpu import read_gpu
@@ -47,6 +47,9 @@ async def _no_store(request, call_next):
 
 @app.on_event("startup")
 def _maybe_autostart() -> None:
+    # Before the worker: the report also preloads the CUDA libraries the ONNX
+    # models need, and it says in the log which runtimes landed on the GPU.
+    accel.report()
     # With AUTO_START set, a container restart (host reboot, or `restart:
     # unless-stopped` after a crash) resumes the worker on its own — the slots
     # then ride out a still-down curator via lease backoff. Lets the agent
@@ -137,6 +140,7 @@ def status():
     s["version"] = FC_VERSION or None
     s["channel"] = FC_CHANNEL or None
     s["revision"] = FC_REVISION or None
+    s["accel"] = accel.LAST or None
     return JSONResponse(s)
 
 
@@ -188,7 +192,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
   width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
  .step:hover{border-color:var(--acc)}
  #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
-  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
+  color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
+ /* The browser's own spin arrows, hidden: the − / + beside each field are the
+    control, styled like the rest of the page (operator, 2026-09-24). */
+ #conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
+ #bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
  .unit{color:var(--mut);font-size:12px;font-weight:600}
  .hint{color:var(--mut);font-size:12px;margin-top:12px}
  .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
@@ -227,6 +235,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  <div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
   a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
  </div>
+ <div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
  <div id=banner class=banner style=display:none>
   curator unreachable — holding work + retrying, resumes on its own (no restart needed)
  </div>
@@ -244,7 +253,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
     <button class=step onclick=setc(1)>+</button>
    </div>
    <div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
+    <button class=step onclick=stepbw(-1)>−</button>
     <input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
+    <button class=step onclick=stepbw(1)>+</button>
     <span class=unit>MB/s</span>
    </div>
   </div>
@@ -312,6 +323,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
   await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
    body:JSON.stringify({value:on})});refresh()
  }
+ function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
+ // Runtimes that did NOT get the GPU, from the startup report. Both fall back
+ // to the CPU without raising, so this banner and the pill are the only place
+ // on this page a slow, CPU-bound agent announces itself.
+ function cpuOnly(s){
+  const a=s.accel||{}
+  return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
+ }
  async function setbw(v){
   v=Math.max(0,parseFloat(v)||0); bw.value=v
   await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
@@ -382,11 +401,17 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
    // unreachable curator; grey when stopped; red with no token.
    let dc='dot', lbl='stopped'
    if(!ok){ dc='dot red'; lbl='no token' }
-   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
+   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
+    if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
    else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
    else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
    dot.className=dc; connlbl.textContent=lbl
    banner.style.display=(st==='running' && !s.queue)?'block':'none'
+   const slow=cpuOnly(s)
+   accelbanner.style.display=slow.length?'block':'none'
+   accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
+    +slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
+    +'. After a driver update, regenerate the CDI spec (agent README).'):''
    queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
   }
  }

agent/fc_agent/client.py

@@ -7,6 +7,8 @@ import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+from . import accel
+
 
 class FcClient:
     def __init__(self, base_url: str, token: str, agent_id: str):
@@ -72,7 +74,10 @@ class FcClient:
     def lease(self, batch_size: int) -> list[dict]:
         r = self.s.post(
             f"{self.base}/api/gpu/jobs/lease",
-            json={"agent_id": self.agent_id, "batch_size": batch_size},
+            json={
+                "agent_id": self.agent_id, "batch_size": batch_size,
+                "accel": accel.summary(),
+            },
             timeout=30,
         )
         r.raise_for_status()
@@ -90,7 +95,9 @@ class FcClient:
         })
 
     def heartbeat(self, job_ids: list[int]) -> None:
-        self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
+        self._post_quiet(
+            "/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
+        )
 
     def fail(self, job_id: int, error: str) -> None:
         self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})

agent/requirements.txt

@@ -1,10 +1,12 @@
 # CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace).
 dghs-imgutils>=0.4
 # GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
-# server-side fallback run.
-onnxruntime-gpu
-# The crop EMBEDDER (concept bag). torch is installed separately in the
-# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic;
+# server-side fallback run. The extras declare the CUDA/cuDNN pip packages its
+# CUDA provider loads (fc_agent/accel.py preloads them) rather than relying on
+# torch happening to install the same ones.
+onnxruntime-gpu[cuda,cudnn]
+# The crop EMBEDDER (concept bag). torch + torchvision are installed separately
+# in the Dockerfile from the cu130 wheel index, so pip never swaps them out;
 # transformers loads whatever SigLIP-family model the server announces.
 transformers>=4.45
 # Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic

alembic/versions/0111_discord_native_ledger.py

@@ -0,0 +1,75 @@
+"""Discord native ingester ledgers — seen and dead-letter, per source.
+
+Milestone 428, #4415. Discord moves off gallery-dl onto the native core, which
+keeps its memory of what a source has already fetched in these two tables
+instead of gallery-dl's archive. Same shape as the SubscribeStar pair.
+
+Revision ID: 0111
+Revises: 0110
+Create Date: 2026-09-24
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0111"
+down_revision = "0110"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "discord_seen_media",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("source_id", sa.Integer(), nullable=False),
+        sa.Column("filehash", sa.String(length=128), nullable=False),
+        sa.Column("post_id", sa.String(length=64), nullable=True),
+        sa.Column(
+            "seen_at", sa.DateTime(timezone=True),
+            server_default=sa.text("now()"), nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_id"], ["source.id"],
+            name=op.f("fk_discord_seen_media_source_id_source"), ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_seen_media")),
+        sa.UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
+    )
+    op.create_index(
+        op.f("ix_discord_seen_media_source_id"), "discord_seen_media", ["source_id"],
+    )
+    op.create_table(
+        "discord_failed_media",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("source_id", sa.Integer(), nullable=False),
+        sa.Column("filehash", sa.String(length=128), nullable=False),
+        sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
+        sa.Column("last_error", sa.Text(), nullable=True),
+        sa.Column(
+            "first_failed_at", sa.DateTime(timezone=True),
+            server_default=sa.text("now()"), nullable=False,
+        ),
+        sa.Column(
+            "last_failed_at", sa.DateTime(timezone=True),
+            server_default=sa.text("now()"), nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_id"], ["source.id"],
+            name=op.f("fk_discord_failed_media_source_id_source"), ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_failed_media")),
+        sa.UniqueConstraint(
+            "source_id", "filehash", name="uq_discord_failed_media_source_id",
+        ),
+    )
+    op.create_index(
+        op.f("ix_discord_failed_media_source_id"), "discord_failed_media", ["source_id"],
+    )
+
+
+def downgrade():
+    op.drop_index(op.f("ix_discord_failed_media_source_id"), table_name="discord_failed_media")
+    op.drop_table("discord_failed_media")
+    op.drop_index(op.f("ix_discord_seen_media_source_id"), table_name="discord_seen_media")
+    op.drop_table("discord_seen_media")

alembic/versions/0112_retire_soft_wip_title.py

@@ -0,0 +1,78 @@
+"""Retire the sketch/doodle WIP title tier — its tags, its review flags, its toggle.
+
+Milestone 430, #4428. The soft tier (#1474) tagged `wip` on any post titled
+sketch / doodle / scribble. Measured on the operator's library it was 6,096 of
+8,876 wip tags, and its conflict audit filled the Gallery's review strip with
+2,086 cards, because most finished art scores >= 0.5 on some content head. A
+"sketch" is usually finished work, so the operator retired it (2026-09-25).
+
+Data:
+
+* A soft tag the operator stood behind is kept and relabelled `manual`: one they
+  confirmed (tag_positive_confirmation), or one whose review flag they resolved
+  while leaving the tag on (the strip's "Keep tag").
+* Every other `wip_title_soft` row is deleted.
+* Unresolved review flags whose tag is no longer on the image are deleted: the
+  question they ask no longer applies. That is the audit's cards, and any older
+  orphan the same way.
+
+Then `import_settings.wip_soft_title_tagging_enabled` is dropped. The downgrade
+restores the column only; deleted tags are not recreated.
+
+Revision ID: 0112
+Revises: 0111
+Create Date: 2026-09-25
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0112"
+down_revision = "0111"
+branch_labels = None
+depends_on = None
+
+
+def retire_soft_wip_tags(conn) -> None:
+    """The data half, on a plain connection, so a test can run it directly."""
+    conn.execute(sa.text("""
+        UPDATE image_tag it SET source = 'manual'
+        WHERE it.source = 'wip_title_soft'
+          AND (
+            EXISTS (
+              SELECT 1 FROM tag_positive_confirmation c
+              WHERE c.image_record_id = it.image_record_id AND c.tag_id = it.tag_id
+            )
+            OR EXISTS (
+              SELECT 1 FROM presentation_review pr
+              WHERE pr.image_record_id = it.image_record_id AND pr.tag_id = it.tag_id
+                AND pr.resolved_at IS NOT NULL
+            )
+          )
+    """))
+    conn.execute(sa.text("DELETE FROM image_tag WHERE source = 'wip_title_soft'"))
+    conn.execute(sa.text("""
+        DELETE FROM presentation_review pr
+        WHERE pr.resolved_at IS NULL
+          AND NOT EXISTS (
+            SELECT 1 FROM image_tag it
+            WHERE it.image_record_id = pr.image_record_id AND it.tag_id = pr.tag_id
+          )
+    """))
+
+
+def upgrade():
+    retire_soft_wip_tags(op.get_bind())
+    op.drop_column("import_settings", "wip_soft_title_tagging_enabled")
+
+
+def downgrade():
+    op.add_column(
+        "import_settings",
+        sa.Column(
+            "wip_soft_title_tagging_enabled",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+    )

alembic/versions/0113_redate_native_images.py

@@ -0,0 +1,60 @@
+"""Re-date images whose post's date arrived after they were linked.
+
+#4431. The native ingesters import a post's media before its record, and the
+date travels in the record (`_post.json`). Every natively downloaded image was
+therefore linked to an undated post and kept its download time in both gallery
+date columns, while the post itself was dated correctly. The importer now
+re-dates a post's images when its record lands; this repairs the images that
+landed before that.
+
+Both columns get back the rules the importer keeps:
+
+* `effective_date` is the primary post's date (left alone when that post has
+  none, as the importer does);
+* `earliest_post_date` is the earliest dated post the image is linked to.
+
+Only rows that differ are written. The downgrade does nothing: the old values
+were download times that no one chose.
+
+Revision ID: 0113
+Revises: 0112
+Create Date: 2026-09-25
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0113"
+down_revision = "0112"
+branch_labels = None
+depends_on = None
+
+
+def redate_images(conn) -> None:
+    """The data step, on a plain connection, so a test can run it directly."""
+    conn.execute(sa.text("""
+        UPDATE image_record ir SET effective_date = p.post_date
+        FROM post p
+        WHERE p.id = ir.primary_post_id
+          AND p.post_date IS NOT NULL
+          AND ir.effective_date IS DISTINCT FROM p.post_date
+    """))
+    conn.execute(sa.text("""
+        UPDATE image_record ir SET earliest_post_date = m.earliest
+        FROM (
+          SELECT ip.image_record_id, MIN(p.post_date) AS earliest
+          FROM image_provenance ip JOIN post p ON p.id = ip.post_id
+          WHERE p.post_date IS NOT NULL
+          GROUP BY ip.image_record_id
+        ) m
+        WHERE m.image_record_id = ir.id
+          AND ir.earliest_post_date IS DISTINCT FROM m.earliest
+    """))
+
+
+def upgrade():
+    redate_images(op.get_bind())
+
+
+def downgrade():
+    pass

alembic/versions/0114_fold_misfiled_attachment_posts.py

@@ -0,0 +1,79 @@
+"""Fold the undated shell posts a misfiled attachment created into the real post.
+
+#4435. A non-image file (an archive, a pdf) downloaded for one of an artist's
+Discord channels was filed under the artist's FIRST Discord source: the
+attachment path looked the source up by (artist, platform), which takes the
+lowest id. That created an undated, url-less post there holding only the
+attachment, and the message's real post record then created the dated post
+under the right source. The importer now uses the source it was downloading
+for; this repairs the pairs it left.
+
+A shell is folded only when all of this holds:
+
+* it has no date and no url, and nothing synthesized it;
+* another post of the same artist, platform and external id HAS a date;
+* no image is linked to the shell (it held an attachment and nothing else).
+
+Its attachments move to the dated post, dropping any the dated post already
+has (same sha256, which the per-post unique forbids twice), and the shell is
+deleted. A shell with no dated twin is left alone: which channel it belongs to
+is not recorded anywhere but the file name. The downgrade does nothing.
+
+Revision ID: 0114
+Revises: 0113
+Create Date: 2026-09-25
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0114"
+down_revision = "0113"
+branch_labels = None
+depends_on = None
+
+_PAIRS = """
+    SELECT DISTINCT ON (shell.id) shell.id AS shell_id, real.id AS real_id
+    FROM post shell
+    JOIN source ss ON ss.id = shell.source_id
+    JOIN post real
+      ON real.artist_id = shell.artist_id
+     AND real.external_post_id = shell.external_post_id
+     AND real.id <> shell.id
+     AND real.post_date IS NOT NULL
+    JOIN source rs ON rs.id = real.source_id AND rs.platform = ss.platform
+    WHERE shell.post_date IS NULL
+      AND shell.post_url IS NULL
+      AND shell.synthesized_by IS NULL
+      AND NOT EXISTS (SELECT 1 FROM image_provenance ip WHERE ip.post_id = shell.id)
+    ORDER BY shell.id, real.id
+"""
+
+
+def fold_misfiled_attachment_posts(conn) -> int:
+    """The data step, on a plain connection, so a test can run it directly.
+    Returns how many shells were folded."""
+    pairs = conn.execute(sa.text(_PAIRS)).all()
+    for shell_id, real_id in pairs:
+        conn.execute(sa.text("""
+            DELETE FROM post_attachment pa
+            WHERE pa.post_id = :shell
+              AND EXISTS (
+                SELECT 1 FROM post_attachment keep
+                WHERE keep.post_id = :real AND keep.sha256 = pa.sha256
+              )
+        """), {"shell": shell_id, "real": real_id})
+        conn.execute(
+            sa.text("UPDATE post_attachment SET post_id = :real WHERE post_id = :shell"),
+            {"shell": shell_id, "real": real_id},
+        )
+        conn.execute(sa.text("DELETE FROM post WHERE id = :shell"), {"shell": shell_id})
+    return len(pairs)
+
+
+def upgrade() -> None:
+    fold_misfiled_attachment_posts(op.get_bind())
+
+
+def downgrade() -> None:
+    pass

alembic/versions/0115_source_display_name.py

@@ -0,0 +1,27 @@
+"""source.display_name — what a source is called on its platform.
+
+#4481. A Discord source's URL is a server id and a channel id, so the
+Subscriptions page could only show two numbers. The Discord ingester already
+loads the server and channel names to walk them; it now keeps them here,
+refreshed on every walk. NULL until a walk has read it.
+
+Revision ID: 0115
+Revises: 0114
+Create Date: 2026-09-28
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0115"
+down_revision = "0114"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("source", "display_name")

alembic/versions/0116_single_color_threshold.py

@@ -0,0 +1,34 @@
+"""The single-color filter's default becomes near-total: 0.95 → 0.995.
+
+#4483. At 0.95 the filter rejected line art: a pencil doodle on white is
+mostly white, and five of Todding's Discord doodles were skipped on import as
+"single color", leaving their posts with text and no image. The predicate now
+samples real pixels instead of a blurred thumbnail, and the default only
+calls an image blank when it essentially is one.
+
+A stored 0.95 is the old default, so it moves with it. Any other value was
+chosen by hand and is left alone.
+
+Revision ID: 0116
+Revises: 0115
+Create Date: 2026-09-28
+
+"""
+from alembic import op
+
+revision = "0116"
+down_revision = "0115"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.alter_column("import_settings", "single_color_threshold", server_default="0.995")
+    op.execute(
+        "UPDATE import_settings SET single_color_threshold = 0.995 "
+        "WHERE single_color_threshold = 0.95"
+    )
+
+
+def downgrade() -> None:
+    op.alter_column("import_settings", "single_color_threshold", server_default="0.95")

alembic/versions/0117_external_link_started_at.py

@@ -0,0 +1,33 @@
+"""external_link.started_at: when a fetch claimed the link.
+
+A worker killed mid-fetch (the Celery hard limit, an OOM, a restart) never
+reaches the code that moves its link out of 'downloading'. The external-link
+sweep returns such a link to the retry path once it has been 'downloading'
+longer than the task's hard limit, which needs the claim time.
+
+A row in 'downloading' with no claim time is treated as stale: no worker
+survives the deploy that adds this column.
+
+Revision ID: 0117
+Revises: 0116
+Create Date: 2026-11-05
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0117"
+down_revision = "0116"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "external_link",
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("external_link", "started_at")

alembic/versions/0118_external_link_recoveries.py

@@ -0,0 +1,32 @@
+"""external_link.recoveries: how many times recovery has retried a dead link.
+
+The daily recovery gives a dead link one more attempt, spaced further apart
+each time, and leaves it dead once the count reaches its limit. A link that
+can never succeed is then downloaded a handful of times in total, not every
+day.
+
+Revision ID: 0118
+Revises: 0117
+Create Date: 2026-11-05
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0118"
+down_revision = "0117"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "external_link",
+        sa.Column(
+            "recoveries", sa.Integer(), nullable=False, server_default=sa.text("0")
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("external_link", "recoveries")

alembic/versions/0119_backup_s3_destination.py

@@ -0,0 +1,43 @@
+"""import_settings: the off-site backup destination (an S3 bucket).
+
+Address, bucket, access key and region are plain settings; the secret key is
+stored sealed with the credential key. All empty means no destination is set
+up, which is every existing install's state.
+
+Revision ID: 0119
+Revises: 0118
+Create Date: 2026-11-06
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0119"
+down_revision = "0118"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("import_settings", sa.Column("backup_s3_endpoint", sa.Text(), nullable=True))
+    op.add_column(
+        "import_settings", sa.Column("backup_s3_bucket", sa.String(255), nullable=True),
+    )
+    op.add_column("import_settings", sa.Column("backup_s3_access_key", sa.Text(), nullable=True))
+    op.add_column(
+        "import_settings", sa.Column("backup_s3_secret_sealed", sa.Text(), nullable=True),
+    )
+    op.add_column(
+        "import_settings",
+        sa.Column(
+            "backup_s3_region", sa.String(64), nullable=False, server_default="us-east-1",
+        ),
+    )
+
+
+def downgrade() -> None:
+    for column in (
+        "backup_s3_region", "backup_s3_secret_sealed", "backup_s3_access_key",
+        "backup_s3_bucket", "backup_s3_endpoint",
+    ):
+        op.drop_column("import_settings", column)

alembic/versions/0120_backup_upload_limit.py

@@ -0,0 +1,30 @@
+"""import_settings: a bandwidth limit for uploads to the off-site backup bucket.
+
+In megabits per second; 0 means no limit. Every install starts at 200 Mbit/s,
+which leaves room on a gigabit link for everything else the server does.
+
+Revision ID: 0120
+Revises: 0119
+Create Date: 2026-11-07
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0120"
+down_revision = "0119"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "import_settings",
+        sa.Column(
+            "backup_upload_limit_mbps", sa.Integer(), nullable=False, server_default="200",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("import_settings", "backup_upload_limit_mbps")

alembic/versions/0121_backup_blob.py

@@ -0,0 +1,46 @@
+"""backup_blob: which library files are already in the off-site bucket.
+
+One row per (destination, sha256) sent. The media backup's work list is every
+image and attachment sha with no row for the current destination, so a run is
+incremental and resumable by construction, and pointing backups at a new
+bucket starts that bucket's copy from the beginning.
+
+backup_run.last_progress_at is the heartbeat of a run that spans hours; the
+stall sweep measures a media run's silence from it, not from its start.
+
+Revision ID: 0121
+Revises: 0120
+Create Date: 2026-11-07
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0121"
+down_revision = "0120"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "backup_blob",
+        sa.Column("target", sa.Text(), nullable=False),
+        sa.Column("sha256", sa.String(64), nullable=False),
+        sa.Column("key", sa.Text(), nullable=False),
+        sa.Column("size_bytes", sa.BigInteger(), nullable=False),
+        sa.Column(
+            "uploaded_at", sa.DateTime(timezone=True), nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.PrimaryKeyConstraint("target", "sha256", name=op.f("pk_backup_blob")),
+    )
+    op.add_column(
+        "backup_run",
+        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("backup_run", "last_progress_at")
+    op.drop_table("backup_blob")

alembic/versions/0122_retire_images_backup.py

@@ -0,0 +1,30 @@
+"""Retire the images tarball backup.
+
+Images and attachments are backed up off-site, one object per file
+(backup_blob). The tarball of /images that this replaces is gone, with its
+retention setting, its path column, and any runs it left.
+
+Revision ID: 0122
+Revises: 0121
+Create Date: 2026-11-08
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0122"
+down_revision = "0121"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute("DELETE FROM backup_run WHERE kind = 'images'")
+    op.drop_column("backup_run", "tar_path")
+    op.drop_column("import_settings", "backup_images_keep_last_n")
+
+
+def downgrade() -> None:
+    op.add_column("import_settings", sa.Column(
+        "backup_images_keep_last_n", sa.Integer(), nullable=False, server_default="3"))
+    op.add_column("backup_run", sa.Column("tar_path", sa.Text(), nullable=True))

backend/app/api/admin.py

@@ -475,6 +475,21 @@ async def trigger_reclaim_attachments():
     return _queued(async_result)
 
 
+@admin_bp.route("/maintenance/import-unimported", methods=["POST"])
+async def trigger_import_unimported():
+    """Import files in the artists' folders that the library has no record of.
+    Body {"dry_run": bool}; dry_run (the DEFAULT) lists what was found, per
+    artist, and changes nothing. dry_run=false queues them as an import batch
+    that attaches each file where it lies. Returns the Celery task id — poll
+    /maintenance/task-result/<id> for the summary."""
+    from ..tasks.admin import import_unimported_files_task
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", True))
+    async_result = import_unimported_files_task.delay(dry_run=dry_run)
+    return _queued(async_result)
+
+
 @admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
 async def trigger_repair_discord_downloads():
     """Clean re-download of the Discord files broken by the `None` naming

backend/app/api/extension.py

@@ -19,7 +19,10 @@ from ..models import AppSetting
 from ..services.extension_service import (
     ExtensionService,
     InvalidUrlError,
+    PosterLookupError,
+    UnknownArtistError,
     UnknownPlatformError,
+    UnknownSourceError,
 )
 from ..services.source_service import KNOWN_PLATFORMS
 from ._responses import error_response as _bad
@@ -87,10 +90,16 @@ async def probe_source():
     url = (request.args.get("url") or "").strip()
     if not url:
         return _bad("invalid_body", detail="url query parameter is required")
+    from .credentials import _get_crypto
+
     async with get_session() as session:
         if not await _ext_key_required(session):
             return _bad("unauthorized", status=401)
-        result = await ExtensionService(session).probe(url)
+        # crypto lets a Discord probe name the server and channel with the
+        # stored token; every other platform ignores it.
+        result = await ExtensionService(session, _get_crypto()).probe(
+            url, names=request.args.get("names") in ("1", "true"),
+        )
     return jsonify(result)
 
 
@@ -102,6 +111,22 @@ async def quick_add_source():
     url = body.get("url")
     if not isinstance(url, str) or not url.strip():
         return _bad("invalid_body", detail="url is required")
+    # Optional: connect the new source to an existing artist (artist_id) or to
+    # the artist of that name (artist_name). A Discord channel names no
+    # creator, so the extension's Add panel always sends one of them.
+    artist_id = body.get("artist_id")
+    if artist_id is not None and (isinstance(artist_id, bool) or not isinstance(artist_id, int)):
+        return _bad("invalid_body", detail="artist_id must be an integer")
+    artist_name = body.get("artist_name")
+    if artist_name is not None and not isinstance(artist_name, str):
+        return _bad("invalid_body", detail="artist_name must be a string")
+    # Patreon is canon: adding a Patreon source to an existing artist can take
+    # the creator's Patreon display name (name only; the slug never moves).
+    use_platform_name = body.get("use_platform_name") is True
+    # #4491: a Discord add can carry the posters picked in the panel.
+    discord_authors = body.get("discord_authors")
+    if discord_authors is not None and not _valid_authors(discord_authors):
+        return _bad("invalid_body", detail="discord_authors must be a list of {id, name}")
 
     from .credentials import _get_crypto
 
@@ -111,7 +136,13 @@ async def quick_add_source():
         try:
             # crypto lets an add resolve the artist's display name via the
             # stored credential (else it falls back to the URL handle). #130.
-            result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
+            result = await ExtensionService(session, _get_crypto()).quick_add_source(
+                url, artist_id=artist_id, artist_name=artist_name,
+                use_platform_name=use_platform_name,
+                discord_authors=discord_authors,
+            )
+        except UnknownArtistError as exc:
+            return _bad("not_found", detail=str(exc), status=404)
         except UnknownPlatformError as exc:
             return _bad(
                 "unknown_platform",
@@ -123,6 +154,59 @@ async def quick_add_source():
     return jsonify(result), (201 if result["created_source"] else 200)
 
 
+def _valid_authors(value) -> bool:
+    """[{id, name}]: the id is what the list keys on, the name only shows it."""
+    return isinstance(value, list) and all(
+        isinstance(a, dict) and isinstance(a.get("id"), str | int)
+        and not isinstance(a.get("id"), bool)
+        and (a.get("name") is None or isinstance(a.get("name"), str))
+        for a in value
+    )
+
+
+@extension_bp.route("/discord/posters", methods=["GET"])
+async def discord_posters():
+    """Who posts in the Discord channel `url` shows, lately, with the likely
+    creator marked (#4491). Reads the channel with the stored token; writes
+    nothing."""
+    url = (request.args.get("url") or "").strip()
+    if not url:
+        return _bad("invalid_body", detail="url query parameter is required")
+    from .credentials import _get_crypto
+
+    async with get_session() as session:
+        if not await _ext_key_required(session):
+            return _bad("unauthorized", status=401)
+        try:
+            result = await ExtensionService(session, _get_crypto()).discord_posters(url)
+        except (InvalidUrlError, UnknownPlatformError) as exc:
+            return _bad("invalid_url", detail=str(exc))
+        except PosterLookupError as exc:
+            return _bad("posters_unavailable", detail=str(exc), status=409)
+    return jsonify(result)
+
+
+@extension_bp.route("/discord/posters", methods=["POST"])
+async def set_discord_posters():
+    """Set the poster list of the source following `url`'s channel."""
+    body = await request.get_json(silent=True)
+    if not isinstance(body, dict) or not isinstance(body.get("url"), str):
+        return _bad("invalid_body", detail="url is required")
+    authors = body.get("authors")
+    if not _valid_authors(authors):
+        return _bad("invalid_body", detail="authors must be a list of {id, name}")
+    async with get_session() as session:
+        if not await _ext_key_required(session):
+            return _bad("unauthorized", status=401)
+        try:
+            result = await ExtensionService(session).set_discord_posters(body["url"], authors)
+        except (InvalidUrlError, UnknownPlatformError, PosterLookupError) as exc:
+            return _bad("invalid_url", detail=str(exc))
+        except UnknownSourceError as exc:
+            return _bad("not_found", detail=str(exc), status=404)
+    return jsonify(result)
+
+
 def _read_manifest_sync() -> dict | None:
     """All the filesystem-touching work for /api/extension/manifest,
     in a sync helper so the async route can dispatch it via

backend/app/api/gpu.py

@@ -245,6 +245,29 @@ async def errors_recover(image_id: int):
 
 # --- Agent (bearer token): lease / submit / heartbeat / fail ------------
 
+
+def _accel_detail(body: dict) -> dict:
+    """The agent's own report of which runtime got the GPU, kept on its roster
+    row so the System view can call a CPU-bound agent degraded (#4410).
+
+    Only a dict of {runtime: {device, error?}} is kept, and each value is
+    reduced to those two short strings: this is written on every lease, by a
+    client the server does not control. An agent that sends nothing (an
+    older build) simply has no `accel`, which reads as not-yet-reported.
+    """
+    raw = body.get("accel")
+    if not isinstance(raw, dict):
+        return {}
+    accel = {}
+    for name, entry in list(raw.items())[:4]:
+        if not isinstance(entry, dict):
+            continue
+        clean = {"device": str(entry.get("device") or "")[:16]}
+        if entry.get("error"):
+            clean["error"] = str(entry["error"])[:200]
+        accel[str(name)[:16]] = clean
+    return {"accel": accel} if accel else {}
+
 @gpu_bp.route("/jobs/lease", methods=["POST"])
 async def lease():
     body = await request.get_json(silent=True) or {}
@@ -267,7 +290,10 @@ async def lease():
             key=f"agent:{agent_id}",
             kind="agent",
             display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
-            details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)},
+            details={
+                "agent_id": agent_id, "last_call": "lease", "leased": len(jobs),
+                **_accel_detail(body),
+            },
         )
         ml = await MLSettings.load(session)
         # image rows for url/mime in one shot
@@ -347,7 +373,10 @@ async def heartbeat():
             key=f"agent:{agent_id}",
             kind="agent",
             display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
-            details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n},
+            details={
+                "agent_id": agent_id, "last_call": "heartbeat", "extended": n,
+                **_accel_detail(body),
+            },
         )
         await session.commit()
     return jsonify({"extended": n})

backend/app/api/settings.py

@@ -57,7 +57,6 @@ _EDITABLE_FIELDS = (
     "translation_target_lang",
     "translation_min_confidence",
     "wip_title_tagging_enabled",
-    "wip_soft_title_tagging_enabled",
 )
 
 # Per-host external-download toggles — all plain booleans, validated uniformly.
@@ -196,12 +195,6 @@ async def update_import_settings():
         return jsonify(
             {"error": "wip_title_tagging_enabled must be a boolean"}
         ), 400
-    if "wip_soft_title_tagging_enabled" in body and not isinstance(
-        body["wip_soft_title_tagging_enabled"], bool
-    ):
-        return jsonify(
-            {"error": "wip_soft_title_tagging_enabled must be a boolean"}
-        ), 400
 
     async with get_session() as session:
         row = await ImportSettings.load(session)

backend/app/api/sources.py

@@ -1,10 +1,13 @@
 """FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
 
+from pathlib import Path
+
 from quart import Blueprint, jsonify, request
 from sqlalchemy import func, select
 
 from ..extensions import get_session
 from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
+from ..services import discord_poster_cleanup as poster_cleanup
 from ..services.artist_membership_service import ArtistMembershipService
 from ..services.artist_membership_service import rescan as membership_rescan
 from ..services.artist_service import ArtistService
@@ -463,3 +466,49 @@ async def adopt_membership():
             return jsonify({"already_tracked": exc.existing_id})
         artist_id = artist.id
     return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
+
+
+# -- Discord: remove posts by people outside the poster list (#4486) --------------
+
+_IMAGES_ROOT = Path("/images")
+
+
+async def _poster_cleanup_preview(source_id: int):
+    async with get_session() as session:
+        return await session.run_sync(
+            lambda s: poster_cleanup.preview(s, source_id=source_id)
+        )
+
+
+@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
+async def other_posters_preview(source_id: int):
+    """What removing other posters' posts would delete. Nothing is touched."""
+    try:
+        projection = await _poster_cleanup_preview(source_id)
+    except LookupError:
+        return _bad("not_found", status=404)
+    except poster_cleanup.PosterCleanupError as exc:
+        return _bad("not_applicable", detail=str(exc), status=409)
+    projection["confirm_token"] = poster_cleanup.confirm_token(projection)
+    return jsonify(projection)
+
+
+@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
+async def other_posters_remove(source_id: int):
+    """Delete them. `confirm` must be the token of the preview the operator saw:
+    a poster list edited since then changes the set, and the token with it."""
+    body = await request.get_json(silent=True) or {}
+    try:
+        projection = await _poster_cleanup_preview(source_id)
+    except LookupError:
+        return _bad("not_found", status=404)
+    except poster_cleanup.PosterCleanupError as exc:
+        return _bad("not_applicable", detail=str(exc), status=409)
+    expected = poster_cleanup.confirm_token(projection)
+    if body.get("confirm") != expected:
+        return _bad("confirm_mismatch", expected=expected)
+    async with get_session() as session:
+        result = await session.run_sync(
+            lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
+        )
+    return jsonify(result)

backend/app/api/system_activity.py

@@ -152,6 +152,8 @@ async def list_runs():
       queue=<name>     filter to one queue
       status=<status>  filter to one status (running/ok/error/timeout/retry)
       task=<substr>    case-insensitive substring match on task_name
+      celery_task_id=<id>  exactly one run — how a page follows a job it
+                           started without having to know its lane
       limit=<int>      default 50, max 200
       before_id=<int>  cursor for keyset pagination
 
@@ -167,6 +169,7 @@ async def list_runs():
     queue = request.args.get("queue")
     status = request.args.get("status")
     task = request.args.get("task")
+    celery_task_id = request.args.get("celery_task_id")
     before_id_raw = request.args.get("before_id")
     before_id = int(before_id_raw) if before_id_raw else None
 
@@ -176,6 +179,8 @@ async def list_runs():
             stmt = stmt.where(TaskRun.queue == queue)
         if status:
             stmt = stmt.where(TaskRun.status == status)
+        if celery_task_id:
+            stmt = stmt.where(TaskRun.celery_task_id == celery_task_id)
         if task:
             # Task names contain literal underscores (download_source,
             # vacuum_analyze) — escape LIKE wildcards so a search for

backend/app/api/system_backup.py

@@ -1,5 +1,5 @@
-"""FC-3h: /api/system/backup — create/list/restore/delete/tag for
-DB + image backups.
+"""/api/system/backup: database dumps, the off-site destination and its media
+backup, restores, test restores, and the run history.
 
 Read endpoints are public on FC (operator-facing internal API; same
 posture as /api/system/activity). Write endpoints take a typed
@@ -20,13 +20,12 @@ system_backup_bp = Blueprint(
     "system_backup", __name__, url_prefix="/api/system/backup",
 )
 
-_KINDS = frozenset({"db", "images"})
+_KINDS = frozenset({"db", "media", "restore", "reconcile", "test"})
 _TAG_MAX_LEN = 64
 _BACKUP_SETTINGS_FIELDS = (
     "backup_db_nightly_enabled",
     "backup_db_nightly_hour_utc",
     "backup_db_keep_last_n",
-    "backup_images_keep_last_n",
 )
 
 
@@ -44,11 +43,13 @@ def _row_to_dict(r: BackupRun) -> dict:
             if r.finished_at and r.started_at else None
         ),
         "sql_path": r.sql_path,
-        "tar_path": r.tar_path,
         "size_bytes": r.size_bytes,
         "error": r.error,
         "restored_from_id": r.restored_from_id,
         "manifest": r.manifest or {},
+        # Where an off-site dump went ({endpoint, bucket, key}); None for a
+        # backup kept on this server.
+        "offsite": (r.manifest or {}).get("offsite"),
     }
 
 
@@ -78,10 +79,6 @@ def _validate_backup_settings_patch(body: dict):
         v = body["backup_db_keep_last_n"]
         if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 365):
             return _bad("invalid_value", detail="backup_db_keep_last_n must be 1..365")
-    if "backup_images_keep_last_n" in body:
-        v = body["backup_images_keep_last_n"]
-        if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 100):
-            return _bad("invalid_value", detail="backup_images_keep_last_n must be 1..100")
     return None
 
 
@@ -96,14 +93,173 @@ async def trigger_db_backup():
     return jsonify({"status": "dispatched"}), 202
 
 
-@system_backup_bp.route("/images", methods=["POST"])
-async def trigger_images_backup():
+@system_backup_bp.route("/media", methods=["POST"])
+async def trigger_media_backup():
+    """Send the images and attachments not yet in the off-site bucket. Refused
+    when no destination is set up; a run already going is left to finish."""
+    from ..services import backup_destination as dest
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        if not dest.describe(row)["configured"]:
+            return _bad("no_destination",
+                        detail="set up an off-site destination before sending files")
+    from ..tasks.backup import backup_media_task
+    backup_media_task.delay(triggered_by="manual")
+    return jsonify({"status": "dispatched"}), 202
+
+
+@system_backup_bp.route("/media", methods=["GET"])
+async def media_coverage():
+    """How much of the library is in the off-site bucket: files and bytes,
+    sent and total, each file counted once however many posts hold it."""
+    from ..services import backup_destination as dest
+    from ..services import backup_media
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        f = dest.fields_of(row)
+        configured = dest.describe(row)["configured"]
+        tid = backup_media.destination_id(f.endpoint, f.bucket) if configured else None
+        counts = await session.run_sync(lambda s: backup_media.coverage(s, tid))
+    return jsonify({"configured": configured, **counts})
+
+
+# ── Restoring from the off-site bucket ──────────────────────────────────────
+
+
+async def _saved_target():
+    """(target, None) for the saved destination, or (None, error response)."""
+    from ..services import backup_destination as dest
+    from ..services.backup_s3 import S3Error
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        try:
+            target = dest.saved_target(row)
+        except S3Error as exc:
+            return None, _bad("destination_unreadable", detail=str(exc))
+    if target is None:
+        return None, _bad("no_destination",
+                          detail="set up an off-site destination before restoring from it")
+    return target, None
+
+
+async def _busy_reason() -> str | None:
+    from ..tasks.backup import backup_busy
+
+    async with get_session() as session:
+        return await session.run_sync(backup_busy)
+
+
+def restore_token(ts: str) -> str:
+    """What the person types to confirm replacing the database with the dump
+    taken at `ts`."""
+    return f"restore-offsite-{ts}"
+
+
+@system_backup_bp.route("/offsite/dumps", methods=["GET"])
+async def list_offsite_dumps():
+    """The database dumps in the off-site bucket, newest first, each with
+    the confirmation a restore of it asks for."""
+    import asyncio
+
+    from ..services import backup_restore
+    from ..services.backup_s3 import S3Error
+
+    target, err = await _saved_target()
+    if err:
+        return err
+    try:
+        dumps = await asyncio.to_thread(backup_restore.list_dumps, target)
+    except S3Error as exc:
+        return _bad("destination_unreachable", detail=str(exc), status=502)
+    return jsonify({
+        "dumps": [
+            {"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "counted": d.counted,
+             "confirm": restore_token(d.ts)}
+            for d in dumps
+        ],
+        # A test restore needs about this much free space on the database's
+        # disk while it runs.
+        "database_bytes": await _database_bytes(),
+    })
+
+
+async def _database_bytes() -> int:
+    from sqlalchemy import text
+
+    async with get_session() as session:
+        return (await session.execute(
+            text("SELECT pg_database_size(current_database())"))).scalar_one()
+
+
+async def _restore_test_blocked() -> str | None:
+    from ..tasks.backup import restore_test_blocked
+
+    async with get_session() as session:
+        return await session.run_sync(restore_test_blocked)
+
+
+@system_backup_bp.route("/offsite/test", methods=["POST"])
+async def restore_test_offsite():
+    """Test a dump from the bucket without touching this install: it is
+    loaded into a temporary database, checked table by table, and dropped.
+    Body: {key}, a dump's key from /offsite/dumps, or {} for the newest.
+    Needs no confirmation; refused while a restore or another test is going."""
     body = await request.get_json(silent=True) or {}
-    tag = _validate_tag(body.get("tag"))
-    if isinstance(tag, tuple):
-        return tag
-    from ..tasks.backup import backup_images_task
-    backup_images_task.delay(tag=tag, triggered_by="manual")
+    key = body.get("key")
+    if key is not None and (not isinstance(key, str) or not key.startswith("database/")):
+        return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
+    _, err = await _saved_target()
+    if err:
+        return err
+    busy = await _restore_test_blocked()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_test_task
+    restore_test_task.delay(dump_key=key)
+    return jsonify({"status": "dispatched"}), 202
+
+
+@system_backup_bp.route("/offsite/restore", methods=["POST"])
+async def restore_from_offsite():
+    """Replace this install's database with a dump from the bucket, then
+    fetch the library's files back. Body: {key, confirm}, where confirm is
+    the dump's token from /offsite/dumps. Refused while a backup or another
+    restore is going."""
+    body = await request.get_json(silent=True) or {}
+    key = body.get("key")
+    if not isinstance(key, str) or not key.startswith("database/"):
+        return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
+    ts = key.removeprefix("database/fabledcurator-").removesuffix(".dump")
+    expected = restore_token(ts)
+    if body.get("confirm") != expected:
+        return _bad("confirm_mismatch", detail=f"confirm must equal {expected!r}",
+                    expected=expected)
+    _, err = await _saved_target()
+    if err:
+        return err
+    busy = await _busy_reason()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_offsite_task
+    restore_offsite_task.delay(dump_key=key)
+    return jsonify({"status": "dispatched"}), 202
+
+
+@system_backup_bp.route("/offsite/fetch-files", methods=["POST"])
+async def fetch_missing_files():
+    """Fetch every library file that is missing from disk back from the
+    bucket, leaving the database as it is."""
+    _, err = await _saved_target()
+    if err:
+        return err
+    busy = await _busy_reason()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_offsite_task
+    restore_offsite_task.delay(files_only=True)
     return jsonify({"status": "dispatched"}), 202
 
 
@@ -175,10 +331,16 @@ async def trigger_restore(run_id: int):
         row = await session.get(BackupRun, run_id)
         if row is None:
             return _bad("not_found", status=404)
-        if row.status != "ok":
+        if (row.manifest or {}).get("offsite"):
             return _bad(
                 "not_restorable",
-                detail=f"source backup status={row.status!r}; only 'ok' rows are restorable",
+                detail=("this dump is kept in the off-site bucket, not on this server, "
+                        "so it can't be restored from here"),
+            )
+        if row.kind != "db" or row.status != "ok" or not row.sql_path:
+            return _bad(
+                "not_restorable",
+                detail="only a finished database dump kept on this server can be restored here",
             )
         expected = f"restore-{row.kind}-{row.id}"
         if supplied != expected:
@@ -187,15 +349,29 @@ async def trigger_restore(run_id: int):
                 detail=f"confirm must equal {expected!r}",
                 expected=expected,
             )
-        kind = row.kind
 
-    if kind == "db":
-        from ..tasks.backup import restore_db_task
-        restore_db_task.delay(source_backup_run_id=run_id)
-    else:  # 'images' (the only other value _KINDS allows via the trigger path)
-        from ..tasks.backup import restore_images_task
-        restore_images_task.delay(source_backup_run_id=run_id)
-    return jsonify({"status": "dispatched", "kind": kind}), 202
+    from ..tasks.backup import restore_db_task
+    restore_db_task.delay(source_backup_run_id=run_id)
+    return jsonify({"status": "dispatched", "kind": "db"}), 202
+
+
+@system_backup_bp.route("/runs/<int:run_id>/test", methods=["POST"])
+async def restore_test_local(run_id: int):
+    """Test a database dump kept on this server, the same way as one in the
+    bucket. Needs no confirmation."""
+    async with get_session() as session:
+        row = await session.get(BackupRun, run_id)
+        if row is None:
+            return _bad("not_found", status=404)
+        if row.kind != "db" or row.status != "ok" or not row.sql_path:
+            return _bad("not_testable",
+                        detail="only a finished database dump on this server can be tested here")
+    busy = await _restore_test_blocked()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_test_task
+    restore_test_task.delay(source_backup_run_id=run_id)
+    return jsonify({"status": "dispatched"}), 202
 
 
 @system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
@@ -216,7 +392,7 @@ async def delete_run(run_id: int):
             )
         from ..services import backup_service
         backup_service.unlink_artifact_files(
-            sql_path=row.sql_path, tar_path=row.tar_path,
+            sql_path=row.sql_path,
             manifest_path=(row.manifest or {}).get("manifest_path"),
         )
         await session.delete(row)
@@ -232,10 +408,121 @@ async def get_settings():
         "backup_db_nightly_enabled": row.backup_db_nightly_enabled,
         "backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
         "backup_db_keep_last_n": row.backup_db_keep_last_n,
-        "backup_images_keep_last_n": row.backup_images_keep_last_n,
     })
 
 
+@system_backup_bp.route("/destination", methods=["GET"])
+async def get_destination():
+    """The off-site destination as the settings screen shows it. The secret
+    key is never returned, only whether one is saved."""
+    from ..services import backup_destination as dest
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        return jsonify(dest.describe(row))
+
+
+@system_backup_bp.route("/destination", methods=["PATCH"])
+async def patch_destination():
+    """Save the destination. Body {endpoint, bucket, access_key, region,
+    secret_key, upload_limit_mbps}; an empty field clears it, and a missing
+    upload_limit_mbps keeps the saved limit. A missing or empty secret_key
+    keeps the saved one, so the screen can save the other fields without the
+    secret being typed again."""
+    from ..services import backup_destination as dest
+
+    body = await request.get_json(silent=True)
+    if not isinstance(body, dict):
+        return _bad("invalid_body", detail="body must be a JSON object")
+    fields, why = dest.clean_fields(body)
+    if fields is None:
+        return _bad("invalid_destination", detail=why)
+    secret = body.get("secret_key")
+    if secret is not None and not isinstance(secret, str):
+        return _bad("invalid_destination", detail="secret_key must be text")
+    limit = body.get("upload_limit_mbps")
+    if limit is not None and (
+        not isinstance(limit, int) or isinstance(limit, bool) or not 0 <= limit <= 100_000
+    ):
+        return _bad("invalid_destination",
+                    detail="upload_limit_mbps must be a whole number from 0 (no limit) up")
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        if limit is not None:
+            row.backup_upload_limit_mbps = limit
+        row.backup_s3_endpoint = fields.endpoint
+        row.backup_s3_bucket = fields.bucket
+        row.backup_s3_access_key = fields.access_key
+        row.backup_s3_region = fields.region
+        if secret and secret.strip():
+            row.backup_s3_secret_sealed = dest.seal(secret.strip())
+        await session.commit()
+        return jsonify(dest.describe(row))
+
+
+@system_backup_bp.route("/destination", methods=["DELETE"])
+async def delete_destination():
+    """Forget the destination, secret included. What was already sent stays in
+    the bucket: FabledCurator never deletes there."""
+    from ..services import backup_destination as dest
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        row.backup_s3_endpoint = None
+        row.backup_s3_bucket = None
+        row.backup_s3_access_key = None
+        row.backup_s3_secret_sealed = None
+        row.backup_s3_region = dest.DEFAULT_REGION
+        await session.commit()
+        return jsonify(dest.describe(row))
+
+
+@system_backup_bp.route("/destination/test", methods=["POST"])
+async def try_destination():
+    """Reach the bucket with the fields in the body (the form as typed, saved
+    or not); a missing secret_key uses the saved one. Lists the bucket and
+    writes nothing. Answers {ok, message} either way: a destination that
+    can't be reached is a result to show, not an API error."""
+    import asyncio
+
+    from ..services import backup_destination as dest
+    from ..services.backup_s3 import S3Error
+    from ..services.credential_crypto import InvalidCredentialBlob
+
+    body = await request.get_json(silent=True) or {}
+    if not isinstance(body, dict):
+        return _bad("invalid_body", detail="body must be a JSON object")
+    fields, why = dest.clean_fields(body)
+    if fields is None:
+        return jsonify({"ok": False, "message": f"Not tested: {why}."})
+    secret = body.get("secret_key")
+    secret = secret.strip() if isinstance(secret, str) else ""
+    if not secret:
+        async with get_session() as session:
+            row = await ImportSettings.load(session)
+            sealed = row.backup_s3_secret_sealed
+        if sealed:
+            try:
+                secret = dest.unseal(sealed)
+            except InvalidCredentialBlob:
+                return jsonify({"ok": False, "message": (
+                    "Not tested: the saved secret key can't be opened on this server. "
+                    "Enter the secret key again."
+                )})
+    target = dest.target_for(fields, secret or None)
+    if target is None:
+        missing = [name for name, v in (
+            ("address", fields.endpoint), ("bucket", fields.bucket),
+            ("access key", fields.access_key), ("secret key", secret),
+        ) if not v]
+        return jsonify({"ok": False, "message": f"Not tested: fill in the {', '.join(missing)}."})
+    try:
+        return jsonify(await asyncio.to_thread(dest.check_target, target))
+    except S3Error as exc:
+        return jsonify({"ok": False, "message": str(exc)})
+
+
 @system_backup_bp.route("/settings", methods=["PATCH"])
 async def patch_settings():
     body = await request.get_json(silent=True)

backend/app/api/system_health.py

@@ -35,6 +35,7 @@ from sqlalchemy import select, text
 from ..config import get_config
 from ..extensions import get_session
 from ..models import ServiceSeen
+from ..services import backup_health
 from ..services.worker_lanes import SWEEP_PERIOD_SECONDS
 
 system_health_bp = Blueprint("system_health", __name__, url_prefix="/api/system")
@@ -72,9 +73,14 @@ assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, (
 PROBE_TIMEOUT_SECONDS = 2.0
 
 _OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
+# Checking in, but working at a fraction of its speed: a GPU agent whose
+# runtimes fell back to the CPU (#4410). Below stale — a part that may have
+# stopped is the more urgent question — and above unknown, because this one
+# IS known to be wrong.
+_DEGRADED = "degraded"
 
 # Worst-first, so an overall verdict is just the max.
-_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3}
+_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4}
 
 
 def _age_state(age_seconds: float) -> str:
@@ -100,6 +106,39 @@ def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
     return f"{name} has not checked in for {ago} — treat it as stopped"
 
 
+def _cpu_runtimes(details: dict) -> list[str]:
+    """The runtimes an agent reported as NOT on the GPU, with why.
+
+    Both torch and onnxruntime fall back to the CPU without raising, so an
+    agent in that state leases, works and checks in exactly like a healthy
+    one. On 2026-09-24 one had been doing so since a driver update left a
+    stale CDI spec; the only sign was a line in the agent's own log.
+    """
+    accel = details.get("accel")
+    if not isinstance(accel, dict):
+        return []
+    out = []
+    for name, entry in sorted(accel.items()):
+        if not isinstance(entry, dict) or entry.get("device") == "cuda":
+            continue
+        why = entry.get("error") or entry.get("device") or "unknown"
+        out.append(f"{name} ({why})")
+    return out
+
+
+def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]:
+    """A roster row's state and its sentence, degraded included."""
+    if state == _OK:
+        cpu = _cpu_runtimes(details)
+        if cpu:
+            return _DEGRADED, (
+                f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. "
+                "After a driver update, regenerate the agent host's CDI spec "
+                "(agent README)."
+            )
+    return state, _describe_learned(name, state, age, details)
+
+
 async def _probe_postgres(session) -> dict:
     started = time.monotonic()
     try:
@@ -175,18 +214,31 @@ async def system_health():
             ).scalars().all()
             for row in rows:
                 age = (now - row.last_seen_at).total_seconds()
-                state = _age_state(age)
+                state, detail = _learned_state(
+                    row.display_name, _age_state(age), age, row.details or {},
+                )
                 parts.append({
                     "key": row.key,
                     "kind": row.kind,
                     "name": row.display_name,
                     "state": state,
-                    "detail": _describe_learned(row.display_name, state, age, row.details or {}),
+                    "detail": detail,
                     "last_seen_at": row.last_seen_at.isoformat(),
                     "first_seen_at": row.first_seen_at.isoformat(),
                     **{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
                 })
 
+            # Whether the backups are keeping up. Read like the rest, and a
+            # failure to read it is reported as unknown rather than raised.
+            try:
+                backup = await session.run_sync(lambda s: backup_health.backup_part(s, now))
+            except Exception as exc:  # noqa: BLE001 — a probe reports, it never raises
+                backup = {"key": "backup", "kind": "backup", "name": "Backups",
+                          "state": _UNKNOWN,
+                          "detail": f"could not be checked: {type(exc).__name__}"}
+            if backup is not None:
+                parts.append(backup)
+
     parts.append(await _probe_redis())
 
     overall = max((p["state"] for p in parts), key=lambda s: _SEVERITY[s], default=_UNKNOWN)

backend/app/beat_scheduler.py

@@ -0,0 +1,77 @@
+"""Celery beat that remembers when each job last ran — from task_run, not a file.
+
+Celery's default PersistentScheduler keeps its memory in a shelve file in the
+working directory. Nothing mounts that directory, so every container recreate
+forgets it, and a scheduler that remembers nothing seeds every entry with
+`last_run_at = now`: each job waits a FULL interval after startup. A daily job
+therefore needs 24 hours without a redeploy to fire. Since the one-container
+image (172e33d) every redeploy restarts beat, and on 2026-09-24 no daily or
+weekly job had run since the 21st (#4408).
+
+task_run already records every task that starts (celery_signals), indexed on
+(task_name, started_at DESC), and prune_task_runs keeps the newest row of each
+task however old it is. So on startup each entry takes its last_run_at from
+there:
+  - a job that is overdue runs at once;
+  - a job that is not due waits only the remainder of its interval;
+  - a job that has never run is due now.
+
+Beat keeps last_run_at in memory from then on, as the default scheduler does;
+only the startup seed changes. If the database cannot be read, the entries keep
+Celery's own default rather than beat failing to start.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+
+from celery.beat import Scheduler
+from sqlalchemy import func, select
+
+log = logging.getLogger(__name__)
+
+# Seed for a job with no recorded run: far enough back that any interval or
+# crontab reads as due.
+NEVER = datetime(2000, 1, 1, tzinfo=UTC)
+
+
+def last_runs(session, task_names: list[str]) -> dict[str, datetime]:
+    """The latest recorded start of each task, by task name."""
+    from .models import TaskRun
+
+    if not task_names:
+        return {}
+    rows = session.execute(
+        select(TaskRun.task_name, func.max(TaskRun.started_at))
+        .where(TaskRun.task_name.in_(sorted(set(task_names))))
+        .group_by(TaskRun.task_name)
+    ).all()
+    return dict(rows)
+
+
+def seed(entries, last: dict[str, datetime]) -> None:
+    """Set each entry's last_run_at from `last`; a task with none is due now."""
+    for entry in entries:
+        entry.last_run_at = last.get(entry.task, NEVER)
+
+
+class TaskRunScheduler(Scheduler):
+    """An in-memory beat seeded from task_run history at startup."""
+
+    def setup_schedule(self):
+        super().setup_schedule()
+        try:
+            from .tasks._sync_engine import sync_session_factory
+
+            with sync_session_factory()() as session:
+                last = last_runs(session, [e.task for e in self.schedule.values()])
+        except Exception:
+            log.exception("beat: could not read task_run; every job waits a full interval")
+            return
+        seed(self.schedule.values(), last)
+        due = sum(1 for e in self.schedule.values() if e.is_due()[0])
+        log.info(
+            "beat: seeded %d job(s) from task_run, %d due now",
+            len(self.schedule), due,
+        )

backend/app/celery_app.py

@@ -4,7 +4,7 @@ Queues:
   import      — filesystem import path (FC-2)
   ml          — WD14 + SigLIP inference (FC-2; runs in the ml-worker image)
   thumbnail   — image and video thumbnail generation (FC-2)
-  download    — gallery-dl tasks (FC-3)
+  download    — source walks: the native ingesters and gallery-dl (FC-3)
   scan        — periodic source checks (FC-3) — kept separate so long imports
                 don't starve the scheduler
   maintenance — recovery sweeps, pHash backfill, GPU-queue coordination, etc.
@@ -62,6 +62,15 @@ def make_celery() -> Celery:
             # can never starve the quick self-healing sweeps (operator-flagged
             # 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
             "backend.app.tasks.maintenance.*": {"queue": "maintenance"},
+            # The one long job in maintenance.py: a whole-library phash
+            # recompute (35 min hard limit; the library was cleared for
+            # re-hashing by migration 0098). On the quick lane it held a
+            # scheduler process for its whole run, and the minute ticks queued
+            # up behind it (2026-09-24: 7 waiting, "all workers busy for 18
+            # minutes"). An exact name wins over the glob above.
+            "backend.app.tasks.maintenance.backfill_phash": {"queue": "maintenance_long"},
+            # Walks a folder tree per artist with undated posts (#4436).
+            "backend.app.tasks.maintenance.date_posts_from_records": {"queue": "maintenance_long"},
             "backend.app.tasks.backup.*": {"queue": "maintenance_long"},
             "backend.app.tasks.admin.*": {"queue": "maintenance_long"},
             "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
@@ -149,6 +158,11 @@ def make_celery() -> Celery:
                 "schedule": 86400.0,  # daily — sweep .part/.partial left by a
                 # download/import killed mid-write (graceful-shutdown fallout)
             },
+            "date-posts-from-records-hourly": {
+                "task": "backend.app.tasks.maintenance.date_posts_from_records",
+                "schedule": 3600.0,  # an empty query once every native post is
+                # dated; otherwise upserts the _post.json a killed walk left (#4436)
+            },
             "backfill-phash-daily": {
                 "task": "backend.app.tasks.maintenance.backfill_phash",
                 "schedule": 86400.0,  # daily — NULL-only, so a no-op once the
@@ -216,11 +230,6 @@ def make_celery() -> Celery:
                 "schedule": 86400.0,  # auto-tag wip/editor process art (#1464);
                 # no-op unless process_auto_apply_enabled (opt-in)
             },
-            "soft-wip-conflict-audit-daily": {
-                "task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
-                "schedule": 86400.0,  # flag ring-loud soft-WIP (sketch/doodle) tags
-                # for review (#1474); no-op with no content heads
-            },
             "prune-presentation-reviews-daily": {
                 "task": "backend.app.tasks.ml.prune_presentation_reviews",
                 "schedule": 86400.0,  # retention: drop resolved review flags >30d
@@ -293,6 +302,10 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.backup.prune_backups",
                 "schedule": 86400.0,  # daily
             },
+            "reconcile-media-backup": {
+                "task": "backend.app.tasks.backup.reconcile_media_backup",
+                "schedule": 604800.0,  # weekly: list the bucket, re-send what's missing
+            },
             # Audit 2026-06-02 — three new per-entity recovery sweeps.
             # Each runs every 5 min like the other recover_stalled_*
             # sweeps; each is a no-op when nothing is stuck.
@@ -353,6 +366,9 @@ def make_celery() -> Celery:
             },
         },
         timezone="UTC",
+        # Beat's memory of when each job last ran comes from task_run, not a
+        # shelve file nothing persists — see beat_scheduler (#4408).
+        beat_scheduler="backend.app.beat_scheduler:TaskRunScheduler",
     )
     # FC-3i: register task_run signal handlers (side-effect import).
     from . import celery_signals  # noqa: F401

backend/app/celery_signals.py

@@ -18,11 +18,18 @@ dark for that interval. Monitoring NEVER breaks the thing it's
 monitoring.
 """
 
+import functools
 import logging
 from datetime import UTC, datetime
 
 from celery.exceptions import SoftTimeLimitExceeded
-from celery.signals import task_failure, task_postrun, task_prerun, task_retry
+from celery.signals import (
+    task_failure,
+    task_postrun,
+    task_prerun,
+    task_retry,
+    worker_ready,
+)
 
 from .models import TaskRun
 from .tasks._sync_engine import sync_session_factory
@@ -53,42 +60,29 @@ _INT32_MIN = -2_147_483_648
 
 
 def _queue_for(task) -> str:
-    """Reverse the task→queue routing from celery_app.task_routes.
-    Keep in sync if task_routes is reordered.
+    """The queue Celery routes this task to — asked of the router itself.
 
-    Audit 2026-06-02: backup/admin/library_audit prefixes were
-    missing here even though task_routes sent all three to
-    'maintenance'. The TaskRun.queue column then lied for those
-    rows (claimed 'default') so per-queue dashboard filters and
-    per-queue threshold overrides silently missed them.
+    This was a hand-kept copy of `celery_app.task_routes`, and it drifted
+    twice (the 2026-06-02 audit, then #4432). Long-lane jobs were recorded as
+    `maintenance`, and translation and gpu_queue runs as `default`, where the
+    5-minute stall sweep failed healthy 35-minute translation runs. The router
+    answers from the same table the broker uses, so the two cannot disagree.
     """
     name = getattr(task, "name", "") or ""
-    if name.startswith("backend.app.tasks.import_file."):
-        return "import"
-    if name.startswith("backend.app.tasks.ml."):
-        return "ml"
-    if name.startswith("backend.app.tasks.thumbnail."):
-        return "thumbnail"
-    if name.startswith((
-        "backend.app.tasks.download.",
-        # External file-host fetches share the download lane (celery_app
-        # routes external.* → download). Mirror it here or TaskRun.queue
-        # lies 'default' for them, so per-queue dashboard filters and the
-        # per-queue threshold override miss them — the same gap the
-        # 2026-06-02 audit fixed for backup/admin/library_audit.
-        "backend.app.tasks.external.",
-    )):
-        return "download"
-    if name.startswith("backend.app.tasks.scan."):
-        return "scan"
-    if name.startswith((
-        "backend.app.tasks.maintenance.",
-        "backend.app.tasks.backup.",
-        "backend.app.tasks.admin.",
-        "backend.app.tasks.library_audit.",
-    )):
-        return "maintenance"
-    return "default"
+    app = getattr(task, "app", None)
+    if app is None:
+        from .celery_app import celery as app
+    return _routed_queue(app, name)
+
+
+@functools.lru_cache(maxsize=1024)
+def _routed_queue(app, name: str) -> str:
+    try:
+        queue = app.amqp.router.route({}, name).get("queue")
+    except Exception:  # noqa: BLE001 — monitoring never breaks the task
+        log.warning("task_run: could not resolve the queue for %s", name)
+        return "default"
+    return getattr(queue, "name", None) or (queue if isinstance(queue, str) else "default")
 
 
 def _target_id_from_args(args) -> int | None:
@@ -225,3 +219,64 @@ def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
         error_message=str(reason) if reason else None,
         retry_count=getattr(request, "retries", 0),
     )
+
+
+def _consumed_queues(consumer) -> set[str]:
+    """The queue names this worker process consumes (its `-Q`), or empty when
+    they can't be read — which makes the boot hook below a no-op, not a guess."""
+    try:
+        return {q.name for q in consumer.task_consumer.queues}
+    except Exception:  # noqa: BLE001 — shape varies across celery versions
+        return set()
+
+
+@worker_ready.connect
+def _on_worker_ready(sender=None, **_):
+    """A lane picks up what its previous process left behind. Only the lane
+    that runs the work does it; the other lanes booting beside it must not,
+    or a lane restarting on its own would disturb work still running.
+    Best-effort: a failure is logged and the worker still starts."""
+    queues = _consumed_queues(sender)
+    if "download" in queues:
+        _clear_download_leftovers()
+    if "maintenance_long" in queues:
+        _resume_offsite_runs()
+
+
+def _resume_offsite_runs() -> None:
+    """The long-maintenance lane carries on an off-site backup or restore that
+    a restart cut off mid-chunk. With nothing queuing its next chunk, the run
+    would read "running" for hours, and block every new run, until the stall
+    sweep ended it."""
+    try:
+        from .tasks.backup import resume_offsite_runs
+
+        resume_offsite_runs()
+    except Exception:  # noqa: BLE001 — never block the worker from starting
+        log.exception("long-maintenance lane boot: resuming off-site runs failed")
+
+
+def _clear_download_leftovers() -> None:
+    """The download lane clears what its previous process left behind.
+
+    A restart SIGKILLs any walk that outlives the stop grace, so its event is
+    never finalized and its platform lock is never released. Both used to wait
+    out timers — a 30-min stall sweep that then blamed the source, and a 27-min
+    lock TTL that stalled every other source on the platform.
+    """
+    booted_at = datetime.now(UTC)
+    try:
+        from .services.download_recovery import interrupt_orphaned_download_events
+        from .services.platform_lock import release_all_platform_locks
+
+        with sync_session_factory()() as session:
+            closed = interrupt_orphaned_download_events(session, booted_at=booted_at)
+            session.commit()
+        released = release_all_platform_locks()
+        if closed or released:
+            log.info(
+                "download lane boot: closed %d orphaned download event(s) as "
+                "interrupted, released %d platform lock(s)", closed, released,
+            )
+    except Exception:  # noqa: BLE001 — never block the worker from starting
+        log.exception("download lane boot recovery failed")

backend/app/frontend.py

@@ -46,6 +46,14 @@ async def serve_extension(filename: str):
 
     The application/x-xpinstall MIME tells Firefox to show its native
     install prompt instead of downloading the file as a blob.
+
+    Caching differs by name, and has to. A versioned name is one build's bytes
+    forever, so it can be cached for good. `fabledcurator-latest.xpi` is ONE
+    URL whose bytes change on every release, and Quart's default for a file
+    is `public, max-age=43200`: a browser that fetched it once reused those
+    bytes for 12 hours, so "install the latest" quietly reinstalled the
+    previous build (operator-flagged 2026-09-25). It is `no-cache` — the ETag
+    still makes an unchanged file a cheap 304.
     """
     if not _XPI_NAME_RE.fullmatch(filename):
         abort(404)
@@ -56,10 +64,11 @@ async def serve_extension(filename: str):
         if not xpis:
             abort(404)
         latest = xpis[-1]
-        return await send_file(
+        resp = await send_file(
             latest, mimetype="application/x-xpinstall",
             attachment_filename=latest.name,
         )
+        return _cache(resp, "no-cache")
     target = (XPI_DIR / filename).resolve()
     try:
         target.relative_to(XPI_DIR)
@@ -67,10 +76,19 @@ async def serve_extension(filename: str):
         abort(404)
     if not target.is_file():
         abort(404)
-    return await send_file(
+    resp = await send_file(
         target, mimetype="application/x-xpinstall",
         attachment_filename=filename,
     )
+    return _cache(resp, "public, max-age=31536000, immutable")
+
+
+def _cache(resp, policy: str):
+    """Set the XPI's Cache-Control, dropping the Expires send_file adds so the
+    two can never disagree."""
+    resp.headers["Cache-Control"] = policy
+    resp.headers.pop("Expires", None)
+    return resp
 
 
 @frontend_bp.route("/")

backend/app/models/__init__.py

@@ -4,10 +4,13 @@ from .app_setting import AppSetting
 from .artist import Artist
 from .artist_membership_suggestion import ArtistMembershipSuggestion
 from .artist_visit import ArtistVisit
+from .backup_blob import BackupBlob
 from .backup_run import BackupRun
 from .base import Base
 from .character_prototype import CcipPrototypeState, CharacterPrototype
 from .credential import Credential
+from .discord_failed_media import DiscordFailedMedia
+from .discord_seen_media import DiscordSeenMedia
 from .download_event import DownloadEvent
 from .external_link import ExternalLink
 from .gpu_job import GpuJob
@@ -53,9 +56,12 @@ __all__ = [
     "Artist",
     "ArtistMembershipSuggestion",
     "ArtistVisit",
+    "BackupBlob",
     "BackupRun",
     "Source",
     "Credential",
+    "DiscordFailedMedia",
+    "DiscordSeenMedia",
     "PatreonFailedMedia",
     "PatreonSeenMedia",
     "SubscribeStarFailedMedia",

backend/app/models/backup_blob.py

@@ -0,0 +1,26 @@
+"""backup_blob: a library file already sent to the off-site bucket.
+
+One row per (destination, sha256). `target` names the destination as
+"<endpoint>/<bucket>", so pointing backups at another bucket leaves these rows
+inert and that bucket's copy starts from the beginning. `key` is where the
+file sits in the bucket (`media/ab/cd/<sha256><ext>`).
+"""
+
+from datetime import datetime
+
+from sqlalchemy import BigInteger, DateTime, String, Text, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class BackupBlob(Base):
+    __tablename__ = "backup_blob"
+
+    target: Mapped[str] = mapped_column(Text, primary_key=True)
+    sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
+    key: Mapped[str] = mapped_column(Text, nullable=False)
+    size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
+    uploaded_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )

backend/app/models/backup_run.py

@@ -1,6 +1,8 @@
 """FC-3h: backup_run — operator-facing artifact record for a backup run.
 
-One row per backup attempt (kind='db' or 'images'). Lifecycle
+One row per run: kind 'db' (a database dump), 'media' (the off-site upload of
+images and attachments, which spans many task chunks), 'restore', 'reconcile'
+(the weekly bucket check) or 'test' (a test restore). Lifecycle
 tracking (started_at/finished_at/duration_ms/exception text) lives
 in task_run from FC-3i — this row records artifact metadata: file
 paths, sizes, tag (retention protection), and restore lineage via
@@ -54,7 +56,6 @@ class BackupRun(Base):
         DateTime(timezone=True), nullable=True, index=True,
     )
     sql_path: Mapped[str | None] = mapped_column(Text, nullable=True)
-    tar_path: Mapped[str | None] = mapped_column(Text, nullable=True)
     size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
     error: Mapped[str | None] = mapped_column(Text, nullable=True)
     manifest: Mapped[dict] = mapped_column(
@@ -62,6 +63,11 @@ class BackupRun(Base):
     )
     # Self-referential FK, unindexed until 0089 (#3300): SET NULL has to find
     # the rows pointing at a deleted run before it can null them.
+    # Heartbeat of a run that spans hours (the media backup): set as it makes
+    # progress, and what the stall sweep measures its silence from.
+    last_progress_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )
     restored_from_id: Mapped[int | None] = mapped_column(
         ForeignKey("backup_run.id", ondelete="SET NULL"),
         nullable=True, index=True,

backend/app/models/discord_failed_media.py

@@ -0,0 +1,36 @@
+"""DiscordFailedMedia — per-source dead-letter ledger of Discord files that
+keep failing to download or validate.
+
+Mirror of SubscribeStarFailedMedia. After `attempts` reaches the dead-letter
+threshold a routine walk skips the file (recovery still retries it); a later
+clean download clears the row. `filehash` is the seen-ledger's key.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class DiscordFailedMedia(Base):
+    __tablename__ = "discord_failed_media"
+    __table_args__ = (
+        UniqueConstraint("source_id", "filehash", name="uq_discord_failed_media_source_id"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    first_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    last_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/discord_seen_media.py

@@ -0,0 +1,34 @@
+"""DiscordSeenMedia — per-source ledger of Discord files already downloaded.
+
+Mirror of SubscribeStarSeenMedia for the native Discord ingester (milestone
+428). `filehash` holds the ingester's per-file key, `<message_id>:<media_id>`:
+the attachment id, or for an embed a hash of its URL path. Not the file's
+position in the message — an edit that removes a file renumbers the rest
+(see `discord_client.MediaItem`). The message record's own gate is the
+synthetic `message:<id>` key in the same column.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class DiscordSeenMedia(Base):
+    __tablename__ = "discord_seen_media"
+    __table_args__ = (
+        UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    seen_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/external_link.py

@@ -80,12 +80,23 @@ class ExternalLink(Base):
         Integer, nullable=False, server_default=text("0")
     )
     last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Times the daily recovery has given this link another attempt after it
+    # went dead. Recovery stops at a limit, so a link that can never succeed
+    # isn't downloaded forever.
+    recoveries: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
     attachment_id: Mapped[int | None] = mapped_column(
         ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
     )
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
+    # When the current fetch claimed the link ('downloading'). The sweep uses it
+    # to recover a link whose worker died mid-fetch.
+    started_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
     completed_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )

backend/app/models/import_settings.py

@@ -9,6 +9,7 @@ from sqlalchemy import (
     CheckConstraint,
     Float,
     Integer,
+    String,
     Text,
     select,
     text,
@@ -39,7 +40,7 @@ class ImportSettings(Base):
     transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
 
     skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
-    single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
+    single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995")
     single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
 
     # Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The
@@ -96,9 +97,19 @@ class ImportSettings(Base):
         Integer, nullable=False, default=14,
         server_default="14",
     )
-    backup_images_keep_last_n: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=3,
-        server_default="3",
+    # Off-site backup destination: an S3 bucket (services/backup_destination).
+    # The secret key is stored sealed with the credential key, never in clear.
+    backup_s3_endpoint: Mapped[str | None] = mapped_column(Text, nullable=True)
+    backup_s3_bucket: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    backup_s3_access_key: Mapped[str | None] = mapped_column(Text, nullable=True)
+    backup_s3_secret_sealed: Mapped[str | None] = mapped_column(Text, nullable=True)
+    backup_s3_region: Mapped[str] = mapped_column(
+        String(64), nullable=False, default="us-east-1",
+        server_default="us-east-1",
+    )
+    # Upload bandwidth to that bucket, in megabits per second; 0 = no limit.
+    backup_upload_limit_mbps: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=200, server_default="200",
     )
 
     # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
@@ -238,13 +249,6 @@ class ImportSettings(Base):
     wip_title_tagging_enabled: Mapped[bool] = mapped_column(
         Boolean, nullable=False, default=True, server_default="true",
     )
-    # Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
-    # a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
-    # are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
-    # precision tier is opt-in (the ring-loud audit surfaces false positives).
-    wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False, server_default="false",
-    )
 
     @classmethod
     async def load(cls, session) -> ImportSettings:

backend/app/models/source.py

@@ -46,6 +46,11 @@ class Source(Base):
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
 
     config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+    # alembic 0115: what the source is called on its platform, where the URL
+    # alone doesn't say — a Discord link is two numbers. The ingester that
+    # walks it refreshes it every walk, so a rename reads as the new name.
+    # NULL until a walk has read it, and on platforms that don't write it.
+    display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
 
     last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     last_error: Mapped[str | None] = mapped_column(Text, nullable=True)

backend/app/scripts/restore.py

@@ -0,0 +1,119 @@
+"""Restore FabledCurator from its off-site bucket, from a shell in the
+container. The Backup screen does the same; this is for an install whose page
+you can't reach yet, or for scripting a rebuild.
+
+    python -m backend.app.scripts.restore list
+    python -m backend.app.scripts.restore database --dump latest --yes
+    python -m backend.app.scripts.restore files
+
+On a brand-new install with no destination saved, name it on the command
+line: --endpoint, --bucket and --access-key, with the secret key in the
+FC_BACKUP_SECRET_KEY environment variable (or typed at the prompt). It is
+checked by listing the bucket, then saved as this install's destination,
+sealed, exactly as the settings page saves it.
+
+`database` replaces the database with the chosen dump here, then hands the
+files to the worker; their progress shows on the Backup screen. `files`
+fetches only what is missing from disk and leaves the database alone.
+"""
+from __future__ import annotations
+
+import argparse
+import getpass
+import os
+import sys
+
+from ..models import ImportSettings
+from ..services import backup_destination, backup_restore
+from ..services.backup_s3 import S3Error
+from ..tasks._sync_engine import sync_session_factory
+
+
+def _human(n: int) -> str:
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if n < 1000 or unit == "TB":
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1000
+    return str(n)
+
+
+def _destination(args) -> object:
+    """The target to restore from: the one named on the command line, saved
+    first, or the one already saved."""
+    session_factory = sync_session_factory()
+    if args.endpoint or args.bucket or args.access_key:
+        fields, why = backup_destination.clean_fields({
+            "endpoint": args.endpoint, "bucket": args.bucket,
+            "access_key": args.access_key, "region": args.region,
+        })
+        if fields is None:
+            raise SystemExit(f"Destination not usable: {why}.")
+        secret = os.environ.get("FC_BACKUP_SECRET_KEY") or getpass.getpass("Secret key: ")
+        target = backup_destination.target_for(fields, secret.strip() or None)
+        if target is None:
+            raise SystemExit("Name the address, bucket, access key and secret key.")
+        check = backup_destination.check_target(target)
+        print(check["message"])
+        if not check["ok"]:
+            raise SystemExit(1)
+        with session_factory() as session:
+            settings = ImportSettings.load_sync(session)
+            limit = settings.backup_upload_limit_mbps
+            backup_restore.keep_destination(session, target, limit)
+        print("Saved as this install's off-site destination.")
+        return target
+    with session_factory() as session:
+        target = backup_destination.saved_target(ImportSettings.load_sync(session))
+    if target is None:
+        raise SystemExit("No off-site destination is saved; name one with "
+                         "--endpoint, --bucket and --access-key.")
+    return target
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m backend.app.scripts.restore",
+                                     description=__doc__.split("\n\n")[0])
+    parser.add_argument("action", choices=("list", "database", "files"))
+    parser.add_argument("--dump", default="latest",
+                        help="the dump's key from `list`, or latest (the default)")
+    parser.add_argument("--yes", action="store_true",
+                        help="confirm replacing this install's database")
+    parser.add_argument("--endpoint")
+    parser.add_argument("--bucket")
+    parser.add_argument("--access-key")
+    parser.add_argument("--region", default=None)
+    args = parser.parse_args(argv)
+
+    try:
+        target = _destination(args)
+        if args.action == "list":
+            for d in backup_restore.list_dumps(target):
+                print(f"{d.key}  {_human(d.size_bytes)}")
+            return 0
+
+        from ..tasks.backup import restore_offsite_task
+        if args.action == "files":
+            restore_offsite_task.delay(files_only=True)
+            print("Queued: the worker fetches every missing file; watch the Backup screen.")
+            return 0
+
+        dump = backup_restore.find_dump(target, args.dump)
+        if not args.yes:
+            print(f"This replaces this install's database with {dump.key} "
+                  f"({_human(dump.size_bytes)}). Run again with --yes to go ahead.")
+            return 2
+        print(f"Restoring {dump.key}; this takes a while for a large library…")
+        result = restore_offsite_task(dump_key=dump.key)
+        if result.get("skipped"):
+            print(f"Not started: {result['skipped']}.")
+            return 1
+        print("The database is restored. The worker is fetching the library's files; "
+              "watch the Backup screen.")
+        return 0
+    except S3Error as exc:
+        print(f"Couldn't restore: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

backend/app/services/archive_extractor.py

@@ -57,6 +57,50 @@ def is_archive(path: Path) -> bool:
     return detect_archive_format(path) is not None
 
 
+# Files that only a program ships: executables and libraries, and the packed
+# data and compiled scripts of the common game engines (Ren'Py, Unity, Godot,
+# RPG Maker, Python builds). An archive holding any of them is a game or app
+# build. Its images are sprites, UI and CGs bundled for the program, not art
+# for the library, so the archive is kept whole as an attachment.
+SOFTWARE_EXTS = frozenset({
+    ".exe", ".msi", ".dll", ".so", ".dylib", ".apk",
+    ".rpa", ".rpyc", ".rpy", ".pyc", ".pyo",
+    ".pak", ".pck", ".unity3d", ".assets", ".ress",
+    ".rgssad", ".rgss2a", ".rgss3a", ".rxdata", ".rvdata", ".rvdata2",
+})
+
+
+def member_names(path: Path) -> list[str]:
+    """Every member name in the archive, read from its directory without
+    decompressing anything. Empty when the archive can't be listed."""
+    try:
+        fmt = detect_archive_format(path)
+        if fmt == "zip":
+            with zipfile.ZipFile(path) as zf:
+                return zf.namelist()
+        if fmt == "rar":
+            import rarfile
+
+            with rarfile.RarFile(path) as rf:
+                return rf.namelist()
+        if fmt == "7z":
+            import py7zr
+
+            with py7zr.SevenZipFile(path, "r") as zf:
+                return zf.getnames()
+    except Exception as exc:
+        log.warning("archive listing failed for %s: %s", path, exc)
+    return []
+
+
+def software_marker(path: Path) -> str | None:
+    """The first member that marks the archive as a software build, or None."""
+    for name in member_names(path):
+        if Path(name).suffix.lower() in SOFTWARE_EXTS:
+            return name
+    return None
+
+
 @contextmanager
 def extract_archive(path: Path):
     """Context manager yielding a list of (member_name, extracted_path)

backend/app/services/artist_service.py

@@ -316,17 +316,32 @@ class ArtistService:
         cleaned = (prefix or "").strip()
         if not cleaned:
             return []
-        like = f"%{cleaned.lower()}%"
-        prefix_like = f"{cleaned.lower()}%"
-        # Rank: exact (0) < prefix (1) < substring (2).
+        low = cleaned.lower()
+        like = f"%{low}%"
+        prefix_like = f"{low}%"
+        # Spacing- and punctuation-insensitive too, so "Tamada Heijun" finds
+        # "TamadaHeijun" and "sabu_art" finds "Sabu Art" — the same creator is
+        # spelled differently on every platform, and the browser extension's
+        # Add panel matches a Discord server name against these (milestone 429).
+        # [[:alnum:]] keeps non-Latin letters; a query with none (all
+        # punctuation) skips this arm rather than matching every artist.
+        squashed = "".join(ch for ch in low if ch.isalnum())
+        name_squashed = func.regexp_replace(func.lower(Artist.name), "[^[:alnum:]]", "", "g")
+        matches = [func.lower(Artist.name).like(like)]
+        if squashed:
+            matches.append(name_squashed.like(f"%{squashed}%"))
+        # Rank: exact (0) < exact ignoring spacing (1) < prefix (2) <
+        # substring (3) < substring ignoring spacing (4).
         rank = case(
-            (func.lower(Artist.name) == cleaned.lower(), 0),
-            (func.lower(Artist.name).like(prefix_like), 1),
-            else_=2,
+            (func.lower(Artist.name) == low, 0),
+            (name_squashed == squashed, 1),
+            (func.lower(Artist.name).like(prefix_like), 2),
+            (func.lower(Artist.name).like(like), 3),
+            else_=4,
         ).label("rank")
         rows = (await self.session.execute(
             select(Artist, rank)
-            .where(func.lower(Artist.name).like(like))
+            .where(or_(*matches))
             .order_by(rank, Artist.name.asc())
             .limit(limit)
         )).all()

backend/app/services/audits/single_color.py

@@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
 2026-07-02 — the import-side filter (Importer._single_color_hit /
 SkipReason.single_color), so what the audit flags and what the import
 skips can never disagree.
+
+It is meant to catch the blank: a placeholder, an error tile, a solid fill.
+Line art is the case it must not catch (#4483): a pencil doodle on white can be
+well over 95% white even at full size, and a smoothing downsample blends its strokes
+into the paper until it measures as blank. So the sample is taken by NEAREST
+(each sampled pixel is a real pixel, strokes keep their contrast), at 256px
+so thin strokes are still hit, and the default threshold is near-total
+(0.995) — a few percent of ink is a drawing, not an empty image.
 """
 
 from PIL import Image
 
-_THUMB_SIZE = (64, 64)
+_THUMB_SIZE = (256, 256)
 
 
 def evaluate(
@@ -20,7 +28,8 @@ def evaluate(
     """True iff the fraction of pixels within `tolerance` (Euclidean RGB
     distance) of the dominant color exceeds `threshold`.
 
-    Downsamples to 64x64 for speed (~4ms regardless of source size).
+    Samples 256x256 by NEAREST (see the module docstring for why not a
+    smoothing resample).
     Alpha channels are stripped; only RGB is considered. Animated images
     use frame 0 (PIL's default after Image.open without seek).
     """
@@ -30,7 +39,7 @@ def evaluate(
     elif im.mode not in ("RGB", "L"):
         im = im.convert("RGB")
     if im.size != _THUMB_SIZE:
-        im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
+        im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST)
     pixels = list(im.getdata())
     if not pixels:
         return False

backend/app/services/backup_destination.py

@@ -0,0 +1,138 @@
+"""The off-site backup destination: an S3 bucket (Fabled Archives, or any S3
+service) that the database dumps and the media blobs are sent to.
+
+The address, bucket, access key and region are plain settings. The secret key
+is sealed at rest with the credential key (`/images/secrets`), the same key
+that seals account logins, and no API ever returns it: the settings answer
+only whether one is set. Because the credential key is kept out of every
+backup, a database dump carries the secret only as ciphertext nobody can open
+without the original server.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from .backup_s3 import S3Error, S3Target, list_objects
+from .credential_crypto import CredentialCrypto, InvalidCredentialBlob
+
+KEY_PATH = Path("/images/secrets/credential_key.b64")
+DEFAULT_REGION = "us-east-1"
+_TEST_LISTING = 5
+
+
+def _crypto() -> CredentialCrypto:
+    return CredentialCrypto(KEY_PATH)
+
+
+def seal(secret: str) -> str:
+    return _crypto().encrypt(secret).decode("ascii")
+
+
+def unseal(sealed: str) -> str:
+    return _crypto().decrypt(sealed.encode("ascii"))
+
+
+@dataclass(frozen=True)
+class DestinationFields:
+    endpoint: str | None
+    bucket: str | None
+    access_key: str | None
+    region: str
+
+
+def clean_fields(body: dict) -> tuple[DestinationFields | None, str | None]:
+    """The destination fields from a request body, trimmed, with empty ones as
+    None; or (None, why) when the address can't be an S3 endpoint."""
+    def text(name: str) -> str | None:
+        v = body.get(name)
+        if v is None:
+            return None
+        if not isinstance(v, str):
+            raise TypeError(name)
+        return v.strip() or None
+
+    try:
+        endpoint = text("endpoint")
+        bucket = text("bucket")
+        access_key = text("access_key")
+        region = text("region") or DEFAULT_REGION
+    except TypeError as exc:
+        return None, f"{exc.args[0]} must be text"
+    if endpoint is not None:
+        u = urlsplit(endpoint)
+        if u.scheme not in ("http", "https") or not u.hostname:
+            return None, ("the address must start with http:// or https:// and name "
+                          "a host, e.g. http://archives.example:8081")
+        if u.query or u.fragment:
+            return None, "the address can't carry a query or a fragment"
+        endpoint = endpoint.rstrip("/")
+    if bucket is not None and "/" in bucket:
+        return None, "the bucket is a name, not a path: put any path in the address"
+    return DestinationFields(endpoint, bucket, access_key, region), None
+
+
+def target_for(fields: DestinationFields, secret: str | None) -> S3Target | None:
+    """The S3 target, or None while any part of it is missing."""
+    if not (fields.endpoint and fields.bucket and fields.access_key and secret):
+        return None
+    return S3Target(fields.endpoint, fields.bucket, fields.access_key, secret,
+                    region=fields.region)
+
+
+def saved_target(settings) -> S3Target | None:
+    """The destination saved in ImportSettings, or None when none is set up.
+    Raises S3Error when the secret can't be opened (the credential key
+    changed since it was saved)."""
+    sealed = settings.backup_s3_secret_sealed
+    if not sealed:
+        return None
+    try:
+        secret = unseal(sealed)
+    except InvalidCredentialBlob:
+        raise S3Error(
+            "The saved secret key can't be opened on this server (the credential "
+            "key changed since it was saved). Enter the secret key again."
+        ) from None
+    return target_for(fields_of(settings), secret)
+
+
+def fields_of(settings) -> DestinationFields:
+    return DestinationFields(
+        settings.backup_s3_endpoint, settings.backup_s3_bucket,
+        settings.backup_s3_access_key, settings.backup_s3_region or DEFAULT_REGION,
+    )
+
+
+def describe(settings) -> dict:
+    """The destination as the settings screen shows it: never the secret."""
+    f = fields_of(settings)
+    return {
+        "endpoint": f.endpoint or "",
+        "bucket": f.bucket or "",
+        "access_key": f.access_key or "",
+        "region": f.region,
+        "secret_set": bool(settings.backup_s3_secret_sealed),
+        "upload_limit_mbps": settings.backup_upload_limit_mbps,
+        "configured": bool(
+            f.endpoint and f.bucket and f.access_key and settings.backup_s3_secret_sealed
+        ),
+    }
+
+
+def check_target(target: S3Target) -> dict:
+    """Reach the bucket by listing it. A listing proves the address, the
+    bucket and both keys at once, and writes nothing, because an Archives key
+    is allowed to add objects but a test object could never be removed."""
+    try:
+        page = list_objects(target, limit=_TEST_LISTING)
+    except S3Error as exc:
+        return {"ok": False, "message": str(exc)}
+    if not page.objects:
+        said = f"Reached {target.bucket} at {target.endpoint}. The bucket is empty."
+    else:
+        more = "+" if page.next_token else ""
+        said = (f"Reached {target.bucket} at {target.endpoint}. It holds "
+                f"{len(page.objects)}{more} object(s), e.g. {page.objects[0].key}.")
+    return {"ok": True, "message": said}

backend/app/services/backup_health.py

@@ -0,0 +1,120 @@
+"""Are the backups keeping up? One row on the System view, and the nav dot.
+
+The backups run on their own at night, so the only way to notice one has
+stopped is to be told. The System view's health endpoint asks `backup_part`,
+which answers with a part like any other: `ok`, or `stale` with the sentence
+that says what is wrong. It warns when
+
+- the newest database backup is more than a day old, or there is none;
+- the newest database backup failed;
+- the weekly check of the off-site bucket found files missing from it, and no
+  media run has finished since to send them again.
+
+No part at all when backups aren't set up (no off-site destination and the
+nightly backup off): someone who chose not to back up is not told so every
+fifteen seconds.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from sqlalchemy import select
+
+from ..models import BackupRun, ImportSettings
+from . import backup_destination
+
+# A nightly backup is due every 24 hours. The hour of slack covers the dump
+# itself and a beat tick that lands a little late.
+DUMP_DUE_SECONDS = 25 * 3600
+
+
+@dataclass(frozen=True)
+class BackupFacts:
+    expected: bool
+    offsite: bool
+    last_dump_at: datetime | None = None
+    last_dump_error: str | None = None
+    missing: int = 0
+
+
+def _age(seconds: float) -> str:
+    hours = int(seconds // 3600)
+    if hours >= 48:
+        return f"{hours // 24} days"
+    if hours >= 1:
+        return f"{hours} h"
+    return f"{max(1, int(seconds // 60))} min"
+
+
+def verdict(facts: BackupFacts, now: datetime) -> dict | None:
+    """The part for these facts, or None when backups aren't set up."""
+    if not facts.expected:
+        return None
+    where = "to the off-site bucket" if facts.offsite else "on this server"
+    problems = []
+    if facts.last_dump_error:
+        problems.append(f"The last database backup failed: {facts.last_dump_error}")
+    if facts.last_dump_at is None:
+        if not facts.last_dump_error:
+            problems.append(f"No database backup has been taken {where} yet")
+    else:
+        age = (now - facts.last_dump_at).total_seconds()
+        if age >= DUMP_DUE_SECONDS:
+            problems.append(f"The newest database backup {where} is {_age(age)} old")
+    if facts.missing:
+        problems.append(
+            f"The bucket check found {facts.missing:,} file(s) missing from the bucket; "
+            "the next media run sends them again"
+        )
+    part = {"key": "backup", "kind": "backup", "name": "Backups"}
+    if problems:
+        return {**part, "state": "stale", "detail": ". ".join(problems)}
+    age = (now - facts.last_dump_at).total_seconds()
+    return {**part, "state": "ok",
+            "detail": f"The newest database backup {where} is {_age(age)} old"}
+
+
+def gather(session) -> BackupFacts:
+    """Read the facts `verdict` judges, from a sync session."""
+    settings = ImportSettings.load_sync(session)
+    offsite = backup_destination.describe(settings)["configured"]
+    expected = offsite or bool(settings.backup_db_nightly_enabled)
+    if not expected:
+        return BackupFacts(expected=False, offsite=False)
+
+    def newest(kind: str, statuses: tuple[str, ...]):
+        return session.execute(
+            select(BackupRun).where(BackupRun.kind == kind)
+            .where(BackupRun.status.in_(statuses))
+            .order_by(BackupRun.started_at.desc()).limit(1)
+        ).scalar_one_or_none()
+
+    last_ok = newest("db", ("ok",))
+    last_done = newest("db", ("ok", "error"))
+    failed = None
+    if last_done is not None and last_done.status == "error":
+        failed = last_done.error or "no reason recorded"
+
+    missing = 0
+    if offsite:
+        check = newest("reconcile", ("ok",))
+        dropped = int((check.manifest or {}).get("dropped") or 0) if check else 0
+        if dropped:
+            sent_since = session.execute(
+                select(BackupRun.id).where(BackupRun.kind == "media")
+                .where(BackupRun.status == "ok")
+                .where(BackupRun.finished_at > check.finished_at).limit(1)
+            ).scalar_one_or_none()
+            missing = 0 if sent_since is not None else dropped
+
+    return BackupFacts(
+        expected=True, offsite=offsite,
+        last_dump_at=(last_ok.finished_at or last_ok.started_at) if last_ok else None,
+        last_dump_error=failed, missing=missing,
+    )
+
+
+def backup_part(session, now: datetime) -> dict | None:
+    return verdict(gather(session), now)

backend/app/services/backup_media.py

@@ -0,0 +1,202 @@
+"""The off-site copy of the library's files: every image and attachment, sent
+once each to the backup bucket as a content-addressed object.
+
+Layout in the bucket, beside the database dumps under `database/`:
+
+    media/ab/cd/<sha256><ext>
+
+`ab` and `cd` are the first four hex digits of the file's sha256, which keeps
+any one prefix small, and the extension keeps the files readable in the bucket
+without FabledCurator. The sha256 is the one recorded at import, so a file
+uploads exactly once however many rows or posts point at it, and a run is
+incremental: its work list is every sha with no `backup_blob` row for the
+current destination. A file removed from the library stays in the bucket until
+the bucket's own retention removes it; FabledCurator never deletes there.
+
+The bucket checks every object against that recorded sha256, so a file whose
+bytes changed on disk since import is refused and reported, never stored under
+the wrong name.
+"""
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import delete, select, text
+from sqlalchemy.dialects.postgresql import insert
+
+from ..models import BackupBlob
+from .backup_s3 import (
+    Pacer,
+    S3Error,
+    S3Target,
+    list_objects,
+    object_present,
+    put_file,
+    put_file_in_parts,
+)
+
+PREFIX = "media/"
+# Files above this size go in parts (S3 refuses a single PUT over 5 GiB, and a
+# part that fails is resent alone instead of the whole file).
+MULTIPART_OVER = 1 << 30
+# The bucket is listed this many keys at a time during a reconcile, with a
+# short rest between pages.
+LIST_PAGE = 1000
+LIST_REST_SECONDS = 0.2
+_EXT = re.compile(r"^\.[a-z0-9]{1,10}$")
+
+
+@dataclass(frozen=True)
+class Blob:
+    sha256: str
+    path: str
+    size_bytes: int
+    mime: str | None
+
+
+def destination_id(endpoint: str, bucket: str) -> str:
+    """How backup_blob names a destination: one copy per endpoint and bucket."""
+    return f"{endpoint.rstrip('/')}/{bucket}"
+
+
+def target_id(target: S3Target) -> str:
+    return destination_id(target.endpoint, target.bucket)
+
+
+def blob_key(sha256: str, path: str) -> str:
+    """The bucket key for a file: `media/ab/cd/<sha256><ext>`. An extension
+    that isn't short and plain is left off, so a key is always safe."""
+    ext = Path(path).suffix.lower()
+    return f"{PREFIX}{sha256[:2]}/{sha256[2:4]}/{sha256}{ext if _EXT.match(ext) else ''}"
+
+
+# Every library file once per sha256: images and attachments together. An
+# attachment's sha can repeat across posts, and an image extracted from an
+# archive is its own file with its own sha.
+_LIBRARY = """
+    SELECT sha256, path, size_bytes, mime FROM image_record
+    UNION ALL
+    SELECT sha256, path, size_bytes, mime FROM post_attachment
+"""
+
+
+def pending(session, tid: str, *, after: str = "", limit: int = 500) -> list[Blob]:
+    """The next `limit` files not yet in the bucket, in sha order after
+    `after`, so a run walks the library once even when some files fail."""
+    rows = session.execute(text(f"""
+        SELECT DISTINCT ON (lib.sha256) lib.sha256, lib.path, lib.size_bytes, lib.mime
+        FROM ({_LIBRARY}) AS lib
+        WHERE lib.sha256 > :after
+          AND NOT EXISTS (
+              SELECT 1 FROM backup_blob b WHERE b.target = :tid AND b.sha256 = lib.sha256
+          )
+        ORDER BY lib.sha256, lib.path
+        LIMIT :limit
+    """), {"after": after, "tid": tid, "limit": limit}).all()
+    return [Blob(r.sha256, r.path, r.size_bytes, r.mime) for r in rows]
+
+
+def coverage(session, tid: str | None) -> dict:
+    """How much of the library is in the bucket: files and bytes, sent and
+    total, each sha counted once."""
+    row = session.execute(text(f"""
+        WITH lib AS (
+            SELECT DISTINCT ON (sha256) sha256, size_bytes FROM ({_LIBRARY}) AS u
+            ORDER BY sha256
+        )
+        SELECT count(*) AS files_total,
+               coalesce(sum(lib.size_bytes), 0) AS bytes_total,
+               count(b.sha256) AS files_sent,
+               coalesce(sum(lib.size_bytes) FILTER (WHERE b.sha256 IS NOT NULL), 0)
+                   AS bytes_sent
+        FROM lib
+        LEFT JOIN backup_blob b ON b.sha256 = lib.sha256 AND b.target = :tid
+    """), {"tid": tid or ""}).one()
+    return {
+        "files_total": row.files_total, "bytes_total": int(row.bytes_total),
+        "files_sent": row.files_sent, "bytes_sent": int(row.bytes_sent),
+    }
+
+
+def send(target: S3Target, blob: Blob, images_root: Path, pacer: Pacer | None) -> str:
+    """Upload one file to its key and return the key. Raises S3Error (or
+    OSError) with the reason when it can't be sent."""
+    path = Path(blob.path)
+    if not path.is_absolute():
+        path = images_root / path
+    if not path.is_file():
+        raise S3Error(f"{path} is not on disk.")
+    size = path.stat().st_size
+    if size != blob.size_bytes:
+        raise S3Error(f"{path} is {size} bytes on disk but {blob.size_bytes} when imported.")
+    key = blob_key(blob.sha256, blob.path)
+    content_type = blob.mime or "application/octet-stream"
+    try:
+        if size > MULTIPART_OVER:
+            put_file_in_parts(target, key, path, sha256=blob.sha256,
+                              content_type=content_type, pacer=pacer)
+        else:
+            put_file(target, key, path, content_type=content_type,
+                     payload_sha256=blob.sha256, pacer=pacer)
+    except S3Error:
+        if object_present(target, key, size):
+            return key          # already there from an earlier run
+        raise
+    return key
+
+
+def record(session, tid: str, blobs: list[tuple[Blob, str]]) -> None:
+    """Note files as sent: (blob, key) pairs. A row already there is kept."""
+    if blobs:
+        session.execute(insert(BackupBlob).values([
+            {"target": tid, "sha256": b.sha256, "key": key, "size_bytes": b.size_bytes}
+            for b, key in blobs
+        ]).on_conflict_do_nothing())
+
+
+def reconcile(session, target: S3Target) -> dict:
+    """Check backup_blob against what the bucket actually holds under
+    `media/`. A row whose object is missing or the wrong size is dropped, so
+    the next run sends that file again; an object the bucket holds for a
+    library file with no row (a restored database, a lost commit) is recorded,
+    so it isn't sent twice. Returns the counts."""
+    tid = target_id(target)
+    held: dict[str, int] = {}
+    token = None
+    while True:
+        page = list_objects(target, PREFIX, LIST_PAGE, continuation=token)
+        held.update((o.key, o.size) for o in page.objects)
+        token = page.next_token
+        if not token:
+            break
+        time.sleep(LIST_REST_SECONDS)
+
+    rows = session.execute(
+        select(BackupBlob.sha256, BackupBlob.key, BackupBlob.size_bytes)
+        .where(BackupBlob.target == tid)
+    ).all()
+    gone = [r.sha256 for r in rows if held.get(r.key) != r.size_bytes]
+    for at in range(0, len(gone), 1000):
+        session.execute(delete(BackupBlob).where(
+            BackupBlob.target == tid, BackupBlob.sha256.in_(gone[at:at + 1000]),
+        ))
+    kept = {r.sha256 for r in rows} - set(gone)
+
+    found = []
+    for r in session.execute(text(f"""
+        SELECT DISTINCT ON (sha256) sha256, path, size_bytes, mime
+        FROM ({_LIBRARY}) AS lib ORDER BY sha256, path
+    """)).all():
+        if r.sha256 in kept:
+            continue
+        blob = Blob(r.sha256, r.path, r.size_bytes, r.mime)
+        key = blob_key(blob.sha256, blob.path)
+        if held.get(key) == blob.size_bytes:
+            found.append((blob, key))
+    for at in range(0, len(found), 1000):
+        record(session, tid, found[at:at + 1000])
+    session.commit()
+    return {"listed": len(held), "dropped": len(gone), "adopted": len(found)}

backend/app/services/backup_restore.py

@@ -0,0 +1,198 @@
+"""Restoring from the off-site bucket: the database from one of its dated
+dumps, then every library file the restored rows name, fetched back to the
+path each row records.
+
+The database comes first because it is the map: each image and attachment row
+holds its file's path and sha256, which is also the file's name in the bucket
+(`media/ab/cd/<sha256><ext>`). A file is checked against that sha256 on
+arrival and only then moved into place, so a restore never leaves a wrong or
+half-written file in the library. A file already on disk at the right size is
+left alone, which makes a restore safe to run again and lets it bring back
+only what a damaged library is missing.
+"""
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import select, text
+
+from ..models import BackupBlob, ImportSettings
+from . import backup_destination
+from .backup_media import _LIBRARY, blob_key
+from .backup_s3 import Pacer, S3Error, S3Target, get_file, list_objects
+
+DUMP_PREFIX = "database/"
+_DUMP_NAME = re.compile(r"^database/fabledcurator-(\d{8}T\d{6}Z)\.dump$")
+_ALEMBIC_INI = Path(__file__).resolve().parents[3] / "alembic.ini"
+_MIGRATE_TIMEOUT_S = 1800
+
+
+@dataclass(frozen=True)
+class Dump:
+    key: str
+    ts: str           # 20261107T134258Z, UTC
+    size_bytes: int
+    counted: bool = False   # its row counts sit beside it, for a test restore
+
+
+def list_dumps(target: S3Target) -> list[Dump]:
+    """Every database dump in the bucket, newest first."""
+    found, keys, token = [], set(), None
+    while True:
+        page = list_objects(target, DUMP_PREFIX, 1000, continuation=token)
+        for o in page.objects:
+            keys.add(o.key)
+            if m := _DUMP_NAME.match(o.key):
+                found.append((o.key, m.group(1), o.size))
+        token = page.next_token
+        if not token:
+            break
+    dumps = [Dump(k, ts, size, counted=k.removesuffix(".dump") + ".counts.json" in keys)
+             for k, ts, size in found]
+    return sorted(dumps, key=lambda d: d.ts, reverse=True)
+
+
+def find_dump(target: S3Target, key: str | None) -> Dump:
+    """The dump at `key`, or the newest when `key` is None or "latest"."""
+    dumps = list_dumps(target)
+    if not dumps:
+        raise S3Error(f"The {target.bucket} bucket holds no database dumps.")
+    if key in (None, "", "latest"):
+        return dumps[0]
+    for d in dumps:
+        if d.key == key:
+            return d
+    raise S3Error(f"{key} is not among the dumps in the {target.bucket} bucket.")
+
+
+def fetch_dump(target: S3Target, dump: Dump, into: Path, pacer: Pacer | None) -> Path:
+    """Download a dump into the folder `into`, checked against its listed
+    size. pg_restore reads it from there."""
+    into.mkdir(parents=True, exist_ok=True)
+    path = into / Path(dump.key).name
+    get_file(target, dump.key, path, size=dump.size_bytes, pacer=pacer)
+    return path
+
+
+def migrate(db_name: str | None = None) -> None:
+    """Bring a restored database up to this install's schema. A dump taken by
+    an older FabledCurator gains the migrations it predates; one from a newer
+    FabledCurator fails here, with alembic's reason, because this install
+    doesn't know its schema. `db_name` names another database on the same
+    server to bring up instead of this install's own."""
+    env = {**os.environ, "DB_NAME": db_name} if db_name else None
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.run(
+            [sys.executable, "-m", "alembic", "-c", str(_ALEMBIC_INI), "upgrade", "head"],
+            cwd=_ALEMBIC_INI.parent, stdout=subprocess.DEVNULL, stderr=err,
+            timeout=_MIGRATE_TIMEOUT_S, env=env,
+        )
+        if proc.returncode != 0:
+            err.seek(0)
+            lines = err.read().decode(errors="replace").strip().splitlines()
+            raise S3Error("The restored database couldn't be brought up to this "
+                          f"version: {lines[-1] if lines else 'alembic failed'}")
+
+
+def keep_destination(session, target: S3Target, limit_mbps: int | None) -> None:
+    """Write this install's destination into the restored settings. The
+    dump's own copy of the secret is sealed with the old server's credential
+    key, which never leaves that server, so it can't be opened here."""
+    settings = ImportSettings.load_sync(session)
+    settings.backup_s3_endpoint = target.endpoint
+    settings.backup_s3_bucket = target.bucket
+    settings.backup_s3_access_key = target.access_key
+    settings.backup_s3_region = target.region
+    settings.backup_s3_secret_sealed = backup_destination.seal(target.secret_key)
+    settings.backup_upload_limit_mbps = limit_mbps
+    session.commit()
+
+
+# ── files ───────────────────────────────────────────────────────────────────
+
+
+@dataclass(frozen=True)
+class LibraryFile:
+    sha256: str
+    paths: tuple[str, ...]     # every row's path for this sha, sorted
+    size_bytes: int
+
+
+def library_files(session, *, after: str = "", limit: int = 200) -> list[LibraryFile]:
+    """The next `limit` distinct files the library names, in sha order after
+    `after`, each with every path a row gives it."""
+    rows = session.execute(text(f"""
+        SELECT lib.sha256, array_agg(DISTINCT lib.path ORDER BY lib.path) AS paths,
+               max(lib.size_bytes) AS size_bytes
+        FROM ({_LIBRARY}) AS lib
+        WHERE lib.sha256 > :after
+        GROUP BY lib.sha256
+        ORDER BY lib.sha256
+        LIMIT :limit
+    """), {"after": after, "limit": limit}).all()
+    return [LibraryFile(r.sha256, tuple(r.paths), r.size_bytes) for r in rows]
+
+
+def sent_keys(session, tid: str, shas: list[str]) -> dict[str, str]:
+    """The bucket key recorded for each sha that was sent to this destination."""
+    if not shas:
+        return {}
+    return dict(session.execute(
+        select(BackupBlob.sha256, BackupBlob.key)
+        .where(BackupBlob.target == tid, BackupBlob.sha256.in_(shas))
+    ).all())
+
+
+def _resolve(path: str, images_root: Path) -> Path:
+    """Where a row's path lands on this server. A path outside the library
+    folder is refused: a restore writes only inside it."""
+    p = Path(path)
+    if not p.is_absolute():
+        p = images_root / p
+    root = Path(*images_root.parts)
+    if ".." in p.parts or not (p == root or root in p.parents):
+        raise S3Error(f"{path} is outside {images_root}, so it isn't restored there.")
+    return p
+
+
+def _present(p: Path, size: int) -> bool:
+    try:
+        return p.is_file() and p.stat().st_size == size
+    except OSError:
+        return False
+
+
+def restore_file(target: S3Target, f: LibraryFile, key: str | None,
+                 images_root: Path, pacer: Pacer | None) -> str:
+    """Put one file back at every path the library gives it. Returns
+    "present" (nothing was missing), "copied" (a path already on disk
+    supplied the others) or "restored" (fetched from the bucket). Raises
+    ObjectMissing when the bucket doesn't hold it, S3Error or OSError
+    otherwise."""
+    places = [_resolve(p, images_root) for p in f.paths]
+    have = [p for p in places if _present(p, f.size_bytes)]
+    need = [p for p in places if p not in have]
+    if not need:
+        return "present"
+    outcome = "copied"
+    if not have:
+        first = need.pop(0)
+        first.parent.mkdir(parents=True, exist_ok=True)
+        get_file(target, key or blob_key(f.sha256, f.paths[0]), first,
+                 sha256=f.sha256, size=f.size_bytes, pacer=pacer)
+        have.append(first)
+        outcome = "restored"
+    for p in need:
+        p.parent.mkdir(parents=True, exist_ok=True)
+        partial = p.with_name(p.name + ".partial")
+        shutil.copyfile(have[0], partial)
+        partial.replace(p)
+    return outcome
+

backend/app/services/backup_s3.py

@@ -0,0 +1,533 @@
+"""A small S3 client for sending backups off the box: AWS Signature Version 4,
+path-style addressing (endpoint/bucket/key), and the calls a backup needs —
+put an object (whole, or in parts when its size isn't known up front), read
+one back, list a prefix.
+
+FabledCurator puts, lists and reads; it never deletes. The bucket keeps the
+history and thins it on its own policy, so a key that may only add and read
+objects is enough, and is the safer key to hand a service: a stolen one cannot
+wipe the history it writes into.
+
+Built on the standard library's http.client rather than boto3: three calls
+don't justify a large dependency, and signing is a pure function the tests
+hold to AWS's own published examples. Every call is synchronous, because the
+uploads run in Celery workers; the API runs the Test destination call in a
+thread.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import http.client
+import os
+import socket
+import ssl
+import time
+import xml.etree.ElementTree as ET
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
+
+EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
+TIMEOUT_SECONDS = 30.0
+# A single object can be a multi-gigabyte video or archive; a slow link still
+# gets this long between bytes before the call gives up.
+TRANSFER_TIMEOUT_SECONDS = 15 * 60
+CHUNK = 1 << 20
+
+
+class S3Error(Exception):
+    """Why a call to the bucket failed, said to the person."""
+
+
+@dataclass(frozen=True)
+class S3Target:
+    endpoint: str          # http(s)://host[:port][/prefix], no bucket
+    bucket: str
+    access_key: str
+    secret_key: str
+    region: str = "us-east-1"
+    verify_tls: bool = True
+
+    def url(self, key: str = "") -> str:
+        base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}"
+        return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/"
+
+
+@dataclass(frozen=True)
+class StoredObject:
+    key: str
+    size: int
+    last_modified: str
+
+
+@dataclass(frozen=True)
+class ListPage:
+    objects: list[StoredObject]
+    next_token: str | None     # pass back as `continuation` for the next page
+
+
+# ── Signing ──────────────────────────────────────────────────────────────────
+
+def _encode(text: str, keep_slash: bool = False) -> str:
+    return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
+
+
+def _hmac(key: bytes, msg: str) -> bytes:
+    return hmac.new(key, msg.encode(), hashlib.sha256).digest()
+
+
+def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
+         access_key: str, secret_key: str, region: str, at: datetime,
+         service: str = "s3") -> dict[str, str]:
+    """The request's headers with Host, x-amz-date, x-amz-content-sha256 and
+    Authorization added. Every header given is signed, Host always."""
+    u = urlsplit(url)
+    stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
+    day = stamp[:8]
+    out = dict(headers)
+    out["Host"] = u.netloc
+    out["x-amz-date"] = stamp
+    out["x-amz-content-sha256"] = payload_sha256
+    signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
+    names = sorted(signed)
+
+    query = sorted(
+        (_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True)
+    )
+    canonical = "\n".join([
+        method.upper(),
+        _encode(unquote(u.path) or "/", keep_slash=True),
+        "&".join(f"{k}={v}" for k, v in query),
+        "".join(f"{n}:{signed[n]}\n" for n in names),
+        ";".join(names),
+        payload_sha256,
+    ])
+    scope = f"{day}/{region}/{service}/aws4_request"
+    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
+                         hashlib.sha256(canonical.encode()).hexdigest()])
+    key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
+                      service), "aws4_request")
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
+                            f"SignedHeaders={';'.join(names)}, Signature={signature}")
+    return out
+
+
+# ── Transport ────────────────────────────────────────────────────────────────
+
+def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection:
+    u = urlsplit(url)
+    if u.scheme == "https":
+        ctx = ssl.create_default_context()
+        if not target.verify_tls:
+            ctx.check_hostname = False
+            ctx.verify_mode = ssl.CERT_NONE
+        return http.client.HTTPSConnection(
+            u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK,
+        )
+    if u.scheme != "http":
+        raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.")
+    return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK)
+
+
+def _request_target(url: str) -> str:
+    u = urlsplit(url)
+    return (u.path or "/") + (f"?{u.query}" if u.query else "")
+
+
+def _reach_error(target: S3Target, exc: Exception) -> S3Error:
+    if isinstance(exc, TimeoutError):
+        return S3Error(f"{target.endpoint} didn't answer in time.")
+    if isinstance(exc, socket.gaierror):
+        return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.")
+    if isinstance(exc, ConnectionRefusedError):
+        return S3Error(
+            f"Couldn't reach {target.endpoint}: nothing is listening there "
+            "(check the port)."
+        )
+    if isinstance(exc, ssl.SSLError):
+        return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).")
+    return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
+
+
+def _says(status: int, reason: str, content: bytes) -> str:
+    """An S3 error response in words: its Code and Message when it has them."""
+    try:
+        root = ET.fromstring(content) if content else None
+        code = (root.findtext("Code") or "") if root is not None else ""
+        message = (root.findtext("Message") or "") if root is not None else ""
+    except ET.ParseError:
+        code, message = "", ""
+    hints = {
+        "SignatureDoesNotMatch": "the secret key is wrong",
+        "InvalidAccessKeyId": "the access key isn't known there",
+        "NoSuchBucket": "there is no such bucket",
+        "AccessDenied": "the key isn't allowed to do that",
+        "XAmzContentSHA256Mismatch": "the file's bytes don't match its recorded checksum",
+    }
+    said = hints.get(code) or message or reason
+    if not code and 400 <= status < 500:
+        # S3 refuses with an <Error><Code> body. A 4xx without one came from
+        # something else at that address — most often a web server on the
+        # default port, when the S3 service listens on a port of its own.
+        said = ("the address answered, but not as S3 — check the scheme and "
+                "the port (an S3 service usually listens on its own port)")
+    return f"HTTP {status}{f' {code}' if code else ''}: {said}."
+
+
+# ── Calls ────────────────────────────────────────────────────────────────────
+
+def file_sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(CHUNK):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def put_file(target: S3Target, key: str, path: Path, *,
+             content_type: str = "application/octet-stream",
+             payload_sha256: str | None = None,
+             pacer: Pacer | None = None,
+             now: datetime | None = None) -> None:
+    """Upload one file as one object, streamed from disk. `payload_sha256` is
+    the file's sha256 when the caller already knows it (the library stores
+    every file's), which saves reading the file twice; the bucket checks the
+    bytes against it, so a file that changed on disk is refused rather than
+    stored under the wrong name."""
+    digest = payload_sha256 or file_sha256(path)
+    size = path.stat().st_size
+    url = target.url(key)
+    headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type},
+                   digest, target.access_key, target.secret_key, target.region,
+                   now or datetime.now(UTC))
+    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
+    try:
+        with open(path, "rb") as f:
+            conn.request("PUT", _request_target(url),
+                         body=pacer.file_chunks(f) if pacer else f, headers=headers)
+            response = conn.getresponse()
+            content = response.read()
+    except (OSError, http.client.HTTPException) as exc:
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+    if response.status // 100 != 2:
+        raise S3Error(f"Upload of {key} refused: "
+                      f"{_says(response.status, response.reason, content)}")
+
+
+class ObjectMissing(S3Error):
+    """The bucket holds nothing at the key asked for."""
+
+
+def get_file(target: S3Target, key: str, path: Path, *,
+             sha256: str | None = None, size: int | None = None,
+             pacer: Pacer | None = None,
+             now: datetime | None = None) -> None:
+    """Download one object to `path`, streamed to disk. The bytes land under a
+    `.partial` name and are renamed into place when complete, so a broken
+    transfer never leaves a truncated file at `path`. With `sha256` (and
+    `size`) given, the bytes are checked on arrival and a copy that doesn't
+    match is thrown away and reported. Raises ObjectMissing when the bucket
+    has no such object."""
+    url = target.url(key)
+    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
+                   target.region, now or datetime.now(UTC))
+    partial = path.with_name(path.name + ".partial")
+    digest = hashlib.sha256()
+    got = 0
+    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
+    try:
+        conn.request("GET", _request_target(url), headers=headers)
+        response = conn.getresponse()
+        if response.status == 404:
+            response.read()
+            raise ObjectMissing(f"{key} is not in the {target.bucket} bucket.")
+        if response.status != 200:
+            raise S3Error(f"Reading {key} refused: "
+                          f"{_says(response.status, response.reason, response.read())}")
+        with open(partial, "wb") as f:
+            while chunk := response.read(CHUNK):
+                if pacer:
+                    pacer.take(len(chunk))
+                digest.update(chunk)
+                got += len(chunk)
+                f.write(chunk)
+        if size is not None and got != size:
+            raise S3Error(f"{key} arrived as {got} bytes, not the {size} expected.")
+        if sha256 and digest.hexdigest() != sha256:
+            raise S3Error(f"{key} arrived with a different fingerprint "
+                          f"({digest.hexdigest()[:12]}…), so it wasn't kept.")
+        os.replace(partial, path)
+    except (OSError, http.client.HTTPException) as exc:
+        partial.unlink(missing_ok=True)
+        raise _reach_error(target, exc) from None
+    except S3Error:
+        partial.unlink(missing_ok=True)
+        raise
+    finally:
+        conn.close()
+
+
+def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
+                 continuation: str | None = None,
+                 now: datetime | None = None) -> ListPage:
+    """One page (up to `limit`) of the objects under `prefix`, ListObjectsV2."""
+    params = {"list-type": "2", "max-keys": str(limit)}
+    if prefix:
+        params["prefix"] = prefix
+    if continuation:
+        params["continuation-token"] = continuation
+    url = f"{target.url()}?{urlencode(params, quote_via=quote)}"
+    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
+                   target.region, now or datetime.now(UTC))
+    conn = _connect(target, url, TIMEOUT_SECONDS)
+    try:
+        conn.request("GET", _request_target(url), headers=headers)
+        response = conn.getresponse()
+        content = response.read()
+    except (OSError, http.client.HTTPException) as exc:
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+    if response.status != 200:
+        raise S3Error(f"Listing {target.bucket} refused: "
+                      f"{_says(response.status, response.reason, content)}")
+    try:
+        root = ET.fromstring(content)
+    except ET.ParseError:
+        raise S3Error(
+            f"{target.endpoint} answered a listing with something that isn't S3's XML."
+        ) from None
+    objects, next_token = [], None
+    for item in root.iter():
+        tag = item.tag.rsplit("}", 1)[-1]
+        if tag == "NextContinuationToken":
+            next_token = item.text or None
+        if tag != "Contents":
+            continue
+        fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
+        objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
+                                    fields.get("LastModified", "")))
+    return ListPage(objects, next_token)
+
+
+# ── Multipart: one object sent in parts ─────────────────────────────────────
+
+# Each part is held in memory while it is sent, so this bounds the memory a
+# streamed upload uses. S3 allows 10,000 parts, which at this size is 640 GiB
+# for one object, and every part but the last must be at least 5 MiB.
+PART_SIZE = 64 * CHUNK
+# A part the bucket couldn't take (it didn't answer, or said it was busy) is
+# sent again after a rest that grows each time: 15 s, then 60 s.
+PART_RETRY_WAITS = (15, 60)
+_BUSY = frozenset({429, 500, 502, 503, 504})
+_MAX_RETRY_AFTER = 300
+
+
+class Pacer:
+    """Holds the bytes a job moves to an average rate, across every request
+    it makes, so a backup or a restore leaves room on the link for everything
+    else."""
+
+    def __init__(self, mbps: int | None):
+        self.rate = mbps * 1_000_000 / 8 if mbps else None   # bytes per second
+        self.sent = 0
+        self.started = time.monotonic()
+
+    def take(self, n: int) -> None:
+        """Count `n` bytes moved, pausing while the job is ahead of its rate."""
+        if self.rate:
+            self.sent += n
+            ahead = self.sent / self.rate - (time.monotonic() - self.started)
+            if ahead > 0:
+                time.sleep(ahead)
+
+    def chunks(self, data: bytes) -> Iterator[memoryview]:
+        view = memoryview(data)
+        for at in range(0, len(view), CHUNK):
+            piece = view[at:at + CHUNK]
+            self.take(len(piece))
+            yield piece
+
+    def file_chunks(self, f) -> Iterator[bytes]:
+        while piece := f.read(CHUNK):
+            self.take(len(piece))
+            yield piece
+
+
+def _send(target: S3Target, method: str, url: str, body: bytes = b"",
+          headers: dict[str, str] | None = None, *, timeout: float = TIMEOUT_SECONDS,
+          pacer: Pacer | None = None,
+          now: datetime | None = None) -> tuple[int, str, http.client.HTTPMessage, bytes]:
+    """Sign and send one request with an in-memory body, paced when a pacer
+    is given. Returns (status, reason, headers, content)."""
+    digest = hashlib.sha256(body).hexdigest()
+    signed = sign(method, url, {"Content-Length": str(len(body)), **(headers or {})},
+                  digest, target.access_key, target.secret_key, target.region,
+                  now or datetime.now(UTC))
+    conn = _connect(target, url, timeout)
+    try:
+        conn.request(method, _request_target(url),
+                     body=pacer.chunks(body) if pacer and body else body, headers=signed)
+        response = conn.getresponse()
+        content = response.read()
+    except (OSError, http.client.HTTPException) as exc:
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+    return response.status, response.reason, response.headers, content
+
+
+def _xml_field(content: bytes, name: str) -> str | None:
+    try:
+        root = ET.fromstring(content)
+    except ET.ParseError:
+        return None
+    for item in root.iter():
+        if item.tag.rsplit("}", 1)[-1] == name:
+            return item.text
+    return None
+
+
+def _is_error_body(content: bytes) -> bool:
+    try:
+        return bool(content) and ET.fromstring(content).tag.rsplit("}", 1)[-1] == "Error"
+    except ET.ParseError:
+        return False
+
+
+def _retry_after(headers: http.client.HTTPMessage | None, default: int) -> int:
+    try:
+        asked = int((headers or {}).get("Retry-After") or default)
+    except ValueError:
+        asked = default
+    return max(0, min(asked, _MAX_RETRY_AFTER))
+
+
+class Upload:
+    """One object written as a series of parts, for data whose size isn't known
+    when it starts, such as a database dump streamed from pg_dump.
+
+    The object appears in the bucket only when `complete()` succeeds. Until
+    then the parts are invisible, so a stream that fails partway leaves
+    nothing behind that looks like a finished backup. `abort()` asks the
+    bucket to drop the parts at once; if the abort is refused or never
+    arrives, S3 services expire an unfinished upload on their own (Fabled
+    Archives after a day)."""
+
+    def __init__(self, target: S3Target, key: str, *,
+                 content_type: str = "application/octet-stream",
+                 pacer: Pacer | None = None):
+        self.target = target
+        self.key = key
+        self.size = 0
+        self.pacer = pacer
+        self._etags: list[str] = []
+        status, reason, _, content = _send(
+            target, "POST", f"{target.url(key)}?uploads", headers={"Content-Type": content_type},
+        )
+        upload_id = _xml_field(content, "UploadId") if status == 200 else None
+        if not upload_id:
+            raise S3Error(f"Starting the upload of {key} refused: "
+                          f"{_says(status, reason, content)}")
+        self.upload_id = upload_id
+
+    def _url(self, **params: str) -> str:
+        query = urlencode({**params, "uploadId": self.upload_id}, quote_via=quote)
+        return f"{self.target.url(self.key)}?{query}"
+
+    @property
+    def parts(self) -> int:
+        return len(self._etags)
+
+    def add(self, data: bytes) -> None:
+        """Send the next part, resting and sending it again when the bucket
+        doesn't answer or says it is busy."""
+        n = len(self._etags) + 1
+        for wait in (*PART_RETRY_WAITS, None):
+            headers = None
+            try:
+                status, reason, headers, content = _send(
+                    self.target, "PUT", self._url(partNumber=str(n)), data,
+                    timeout=TRANSFER_TIMEOUT_SECONDS, pacer=self.pacer,
+                )
+            except S3Error as exc:
+                if wait is None:
+                    raise S3Error(f"Part {n} of {self.key}: {exc}") from None
+                time.sleep(wait)
+                continue
+            etag = headers.get("ETag") if status == 200 else None
+            if etag:
+                self._etags.append(etag)
+                self.size += len(data)
+                return
+            if status not in _BUSY or wait is None:
+                raise S3Error(f"Part {n} of {self.key} refused: "
+                              f"{_says(status, reason, content)}")
+            time.sleep(_retry_after(headers, wait))
+
+    def complete(self) -> None:
+        """Join the parts into the object. S3 can refuse a completion with an
+        error body under a 200 status, so the body is read too."""
+        if not self._etags:
+            raise S3Error(f"Nothing was sent for {self.key}.")
+        body = "<CompleteMultipartUpload>" + "".join(
+            f"<Part><PartNumber>{n}</PartNumber><ETag>{etag}</ETag></Part>"
+            for n, etag in enumerate(self._etags, start=1)
+        ) + "</CompleteMultipartUpload>"
+        status, reason, _, content = _send(
+            self.target, "POST", self._url(), body.encode(),
+            headers={"Content-Type": "application/xml"}, timeout=TRANSFER_TIMEOUT_SECONDS,
+        )
+        if status != 200 or _is_error_body(content):
+            raise S3Error(f"Finishing the upload of {self.key} refused: "
+                          f"{_says(status, reason, content)}")
+
+    def abort(self) -> None:
+        """Drop the parts sent so far. Never raises: it runs while another
+        failure is already being reported, and the bucket expires the parts on
+        its own if this doesn't reach it."""
+        try:
+            _send(self.target, "DELETE", self._url())
+        except Exception:  # noqa: BLE001 — best effort, see above
+            pass
+
+
+def put_file_in_parts(target: S3Target, key: str, path: Path, *, sha256: str,
+                      content_type: str = "application/octet-stream",
+                      pacer: Pacer | None = None) -> int:
+    """Upload a large file as a multipart object, read from disk a part at a
+    time. The whole file's sha256 is computed on the way and must equal
+    `sha256` before the upload is completed; otherwise it is aborted, so a file
+    that changed on disk is never stored under its old name. Returns the
+    number of bytes sent."""
+    upload = Upload(target, key, content_type=content_type, pacer=pacer)
+    digest = hashlib.sha256()
+    try:
+        with open(path, "rb") as f:
+            while part := f.read(PART_SIZE):
+                digest.update(part)
+                upload.add(part)
+        if digest.hexdigest() != sha256:
+            raise S3Error(f"{path} no longer matches its recorded checksum; not stored.")
+        upload.complete()
+    except BaseException:
+        upload.abort()
+        raise
+    return upload.size
+
+
+def object_present(target: S3Target, key: str, size: int) -> bool:
+    """Whether `key` is already in the bucket at `size` bytes. Used when a put
+    is refused: an append-only bucket refuses to replace an object that is
+    already there, which for a content-addressed key means the work is done."""
+    try:
+        page = list_objects(target, prefix=key, limit=1)
+    except S3Error:
+        return False
+    return any(o.key == key and o.size == size for o in page.objects)

backend/app/services/backup_service.py

@@ -1,12 +1,11 @@
 """FC-3h: first-class backup/restore service for FC.
 
-Two independent backup kinds:
-  - 'db'     — pg_dump only; fast; nightly via Beat (settings-gated)
-  - 'images' — tar+zstd of /images; slow; manual trigger only
-
-Files live under <images_root>/_backups/. Each backup writes:
-  fc_<kind>_<ts>.{sql|tar.zst}        — the artifact
-  fc_<kind>_<ts>.json                 — manifest (kind/tag/triggered_by)
+The database dump (pg_dump), nightly via Beat when the setting is on. With an
+off-site destination set up, the dump streams straight to that bucket
+(backup_db_offsite), and images and attachments go there too (backup_media).
+Without one, the dump is written under <images_root>/_backups/:
+  fc_db_<ts>.dump                     — the dump (custom format)
+  fc_db_<ts>.json                     — manifest (tag, trigger, row counts)
 
 Service functions are sync (subprocess-bound). Celery tasks in
 backend.app.tasks.backup wrap each one with task_run-tracked
@@ -15,33 +14,19 @@ lifecycle + soft/hard time limits + retention bookkeeping.
 from __future__ import annotations
 
 import json
+import logging
 import os
 import shutil
 import subprocess
 import tempfile
+import time
+from contextlib import contextmanager
 from datetime import UTC, datetime
 from pathlib import Path
 
-_BACKUPS_DIRNAME = "_backups"
+log = logging.getLogger(__name__)
 
-# Excluded from the images tarball, and each for its own reason (#4233, #4234):
-#
-#   _backups    — the archive would otherwise contain every previous archive.
-#                 This is not hypothetical: the 2026-05-23/24 runs, taken before
-#                 this exclude existed, grew 43G -> 107G -> ... -> 2123G as each
-#                 swallowed its predecessors, and cost 4.3T of the images
-#                 filesystem until they were reclaimed on 2026-09-21.
-#   _quarantine — holds files deliberately pulled OUT of the library.
-#   secrets     — `credential_key.b64`, the key that decrypts the stored
-#                 Patreon/SubscribeStar session credentials.
-#   cookies     — those session cookies themselves.
-#
-# The last two are the ones worth stating plainly: an images tarball is a media
-# archive, and a media archive that carries the key to the operator's accounts
-# is a credential leak wearing a backup's name. Encryption at rest buys nothing
-# when the key rides along in the same file. A restore therefore does NOT
-# re-establish credentials — you sign in again, which is the correct outcome.
-_IMAGES_EXCLUDED_DIRNAMES = ("_backups", "_quarantine", "secrets", "cookies")
+_BACKUPS_DIRNAME = "_backups"
 
 # Subprocess-level guardrails BEYOND the Celery soft_time_limit. The Celery
 # soft limit signals the Python process; subprocess.Popen in a blocking syscall
@@ -51,9 +36,7 @@ _IMAGES_EXCLUDED_DIRNAMES = ("_backups", "_quarantine", "secrets", "cookies")
 # matters because an NFS D-state hang defeats even Celery's SIGKILL (the failure
 # that wedged the maintenance lane for hours, #739).
 #   backup_db_task:     soft=1800s / hard=2100s  → 1700s
-#   backup_images_task: soft=21600s / hard=23400s → 21000s
 _DB_SUBPROCESS_TIMEOUT_S = 1700            # ~28 min, under the 30-min DB soft limit
-_IMAGES_SUBPROCESS_TIMEOUT_S = 21000       # ~5.8 hr, under the 6-hr images soft limit
 # Grace after SIGKILL to reap the child. If it can't be reaped in this window
 # (an uninterruptible NFS D-state — the failure mode that wedged the
 # concurrency-1 maintenance lane for hours, operator-flagged 2026-06-07), we
@@ -62,6 +45,85 @@ _IMAGES_SUBPROCESS_TIMEOUT_S = 21000       # ~5.8 hr, under the 6-hr images soft
 _KILL_REAP_GRACE_S = 10
 
 
+# Tables whose rows stay out of every dump. `credential` holds account logins,
+# sealed with the key in /images/secrets: a restored install keeps the table,
+# empty, and you sign in to each account again.
+DATA_LEFT_OUT = ("credential",)
+
+# Every database dump, local or off-site: custom format (-Fc, compressed and
+# restored with pg_restore), without ownership or grants so it loads into any
+# install.
+_PG_DUMP = ["pg_dump", "--no-owner", "--no-acl", "-Fc",
+            *(f"--exclude-table-data={t}" for t in DATA_LEFT_OUT)]
+
+
+def table_counts(conn) -> dict[str, int]:
+    """How many rows each table in the public schema holds, read on a psycopg
+    connection, in the order of the tables' names."""
+    from psycopg import sql
+
+    names = [r[0] for r in conn.execute(
+        "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename")]
+    return {
+        n: conn.execute(sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(n))).fetchone()[0]
+        for n in names
+    }
+
+
+def database_counts(conn) -> dict:
+    """{"alembic": the schema's revision, "tables": {name: rows}}."""
+    has_version = conn.execute("SELECT to_regclass('public.alembic_version')").fetchone()[0]
+    alembic = (conn.execute("SELECT version_num FROM alembic_version").fetchone()
+               if has_version else None)
+    return {"alembic": alembic[0] if alembic else None, "tables": table_counts(conn)}
+
+
+def read_counts(db_url: str) -> dict:
+    """database_counts for the database at `db_url`, as it is now."""
+    import psycopg
+
+    with psycopg.connect(_libpq_url(db_url)) as conn:
+        return database_counts(conn)
+
+
+class DumpSnapshot:
+    """A read-only transaction whose snapshot pg_dump dumps from (it is handed
+    over with --snapshot), so rows counted here are exactly the rows the dump
+    holds. A test restore checks every table against these counts."""
+
+    def __init__(self, conn):
+        self.conn = conn
+        self.id = conn.execute("SELECT pg_export_snapshot()").fetchone()[0]
+
+    def counts(self) -> dict | None:
+        """The counts, or None when they can't be read: the dump is good
+        without them, and a test restore then says none were recorded."""
+        try:
+            return self._counts()
+        except Exception:  # noqa: BLE001 — see above
+            log.exception("backup: the dump's row counts couldn't be read")
+            return None
+
+    def _counts(self) -> dict:
+        counts = database_counts(self.conn)
+        for t in DATA_LEFT_OUT:
+            if t in counts["tables"]:
+                counts["tables"][t] = 0      # the dump holds the table, not its rows
+        return counts
+
+
+@contextmanager
+def dump_snapshot(db_url: str):
+    """Open the snapshot a dump is taken from. It stays open until the dump
+    is finished: pg_dump can only take it up while it is."""
+    import psycopg
+
+    with psycopg.connect(_libpq_url(db_url)) as conn:
+        conn.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        conn.read_only = True
+        yield DumpSnapshot(conn)
+
+
 def _run_bounded(cmd: list[str], timeout: int) -> None:
     """subprocess.run(check=True, timeout) whose reaper can't itself hang.
 
@@ -117,7 +179,7 @@ def _file_size_or_none(path: Path) -> int | None:
 def _write_manifest(
     out_dir: Path, *, kind: str, ts: str,
     tag: str | None, triggered_by: str,
-    artifact_path: Path,
+    artifact_path: Path, counts: dict | None = None,
 ) -> Path:
     manifest = {
         "kind": kind,
@@ -127,6 +189,8 @@ def _write_manifest(
         "created_at": datetime.now(UTC).isoformat(),
         "artifact_path": str(artifact_path),
     }
+    if counts is not None:
+        manifest["counts"] = counts
     mf = out_dir / f"fc_{kind}_{ts}.json"
     mf.write_text(json.dumps(manifest, indent=2))
     return mf
@@ -152,104 +216,215 @@ def backup_db(
     os.close(fd)
     tmp_path = Path(tmp_name)
     try:
-        _run_bounded(
-            [
-                "pg_dump", "--no-owner", "--no-acl", "-Fc",
-                "-f", str(tmp_path), _libpq_url(db_url),
-            ],
-            _DB_SUBPROCESS_TIMEOUT_S,
-        )
+        with dump_snapshot(db_url) as snap:
+            _run_bounded(
+                _gently([*_PG_DUMP, f"--snapshot={snap.id}", "-f", str(tmp_path),
+                         _libpq_url(db_url)]),
+                _DB_SUBPROCESS_TIMEOUT_S,
+            )
+            counts = snap.counts()
         shutil.move(str(tmp_path), str(sql_path))
     finally:
         if tmp_path.exists():
             tmp_path.unlink(missing_ok=True)
     manifest_path = _write_manifest(
         out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
-        artifact_path=sql_path,
+        artifact_path=sql_path, counts=counts,
     )
     return {
         "kind": "db",
         "ts": ts,
         "sql_path": str(sql_path),
-        "tar_path": None,
         "manifest_path": str(manifest_path),
         "size_bytes": _file_size_or_none(sql_path),
+        "counts": counts,
     }
 
 
-def backup_images(
-    *, images_root: Path,
-    tag: str | None = None, triggered_by: str = "manual",
-) -> dict:
-    """Run tar --zstd over images_root; write .tar.zst + manifest."""
+class DumpFailed(Exception):
+    """pg_dump exited with an error; the message is its last line of output."""
+
+
+def offsite_dump_key(ts: str) -> str:
+    """Where a database dump lands in the off-site bucket. Dated names sit
+    side by side under one prefix, which is what lets the bucket's retention
+    thin them by age."""
+    return f"database/fabledcurator-{ts}.dump"
+
+
+def _gently(cmd: list[str]) -> list[str]:
+    """`cmd` at low CPU priority, and low disk priority where `ionice` is
+    installed, so a long dump leaves the server responsive."""
+    io = ["ionice", "-c", "2", "-n", "7"] if shutil.which("ionice") else []
+    return ["nice", "-n", "10", *io, *cmd]
+
+
+def _last_line(f) -> str | None:
+    f.seek(0)
+    lines = f.read().decode(errors="replace").strip().splitlines()
+    return lines[-1].strip() if lines else None
+
+
+def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) -> dict:
+    """Stream `pg_dump` straight into the off-site bucket, in parts, with no
+    copy staged on this server. The object appears only once pg_dump has
+    exited cleanly and every part has arrived; a dump that fails partway
+    leaves nothing in the bucket, and raises DumpFailed with pg_dump's own
+    last line. Returns a dict for the caller to record in BackupRun, with
+    `counts`: each table's rows in the dump, for the caller to store beside
+    it."""
     ts = _now_ts()
-    out_dir = _backups_dir(images_root)
-    tar_path = out_dir / f"fc_images_{ts}.tar.zst"
-    # No local-temp here (the archive is hundreds of GB — it can't stage in
-    # /tmp), but bounded-kill still applies so a tar wedged on NFS fails fast
-    # rather than holding the lane for hours.
-    _run_bounded(
-        [
-            "tar", "--zstd", "-cf", str(tar_path),
-            "-C", str(images_root.parent), images_root.name,
-            *(
-                f"--exclude={images_root.name}/{name}"
-                for name in _IMAGES_EXCLUDED_DIRNAMES
-            ),
-        ],
-        _IMAGES_SUBPROCESS_TIMEOUT_S,
-    )
-    manifest_path = _write_manifest(
-        out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by,
-        artifact_path=tar_path,
-    )
+    with dump_snapshot(db_url) as snap:
+        result = _stream_dump(db_url, target, ts, limit_mbps, snap.id)
+        result["counts"] = snap.counts()
+    return result
+
+
+def _stream_dump(db_url: str, target, ts: str, limit_mbps: int | None, snapshot: str) -> dict:
+    from .backup_s3 import PART_SIZE, Pacer, Upload
+
+    key = offsite_dump_key(ts)
+    deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.Popen(
+            _gently([*_PG_DUMP, f"--snapshot={snapshot}", _libpq_url(db_url)]),
+            stdout=subprocess.PIPE, stderr=err,
+        )
+        upload = None
+        try:
+            upload = Upload(target, key, pacer=Pacer(limit_mbps))
+            while part := proc.stdout.read(PART_SIZE):
+                if time.monotonic() > deadline:
+                    raise subprocess.TimeoutExpired(proc.args, _DB_SUBPROCESS_TIMEOUT_S)
+                upload.add(part)
+            proc.stdout.close()
+            proc.wait(timeout=_KILL_REAP_GRACE_S * 6)
+            if proc.returncode != 0:
+                raise DumpFailed(_last_line(err) or f"pg_dump exited with {proc.returncode}")
+            upload.complete()
+        except BaseException:
+            if proc.poll() is None:
+                proc.kill()
+                try:
+                    proc.wait(timeout=_KILL_REAP_GRACE_S)
+                except subprocess.TimeoutExpired:
+                    pass  # unkillable (D-state): abandon the reap, fail fast
+            if upload is not None:
+                upload.abort()
+            raise
     return {
-        "kind": "images",
+        "kind": "db",
         "ts": ts,
-        "sql_path": None,
-        "tar_path": str(tar_path),
-        "manifest_path": str(manifest_path),
-        "size_bytes": _file_size_or_none(tar_path),
+        "key": key,
+        "bucket": target.bucket,
+        "endpoint": target.endpoint,
+        "parts": upload.parts,
+        "size_bytes": upload.size,
     }
 
 
+class RestoreFailed(Exception):
+    """psql or pg_restore exited with an error; the message is its last line."""
+
+
+# A restore waits this long for other connections to let go of the tables it
+# replaces, then gives up with Postgres' own reason instead of queueing every
+# other query behind it.
+_RESTORE_LOCK_TIMEOUT = "60s"
+# pg_restore of a large library (vector tables, rebuilt indexes) runs well past
+# a dump's length; it is bounded on its own.
+_RESTORE_SUBPROCESS_TIMEOUT_S = 3 * 3600
+
+
+def _run_reporting(cmd: list[str], timeout: int, env: dict | None = None) -> None:
+    """Run `cmd`; raise RestoreFailed with the last line it printed when it
+    exits non-zero."""
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err, timeout=timeout,
+                              env=env)
+        if proc.returncode != 0:
+            raise RestoreFailed(_last_line(err) or f"{cmd[0]} exited with {proc.returncode}")
+
+
+# Every other connection to this database: the web server's, the other
+# workers'. Ended before the tables are dropped, so none holds a lock the drop
+# waits on, and again once the restore is in, so none keeps statements it
+# prepared against the old tables. Each reconnects on its next query.
+_END_OTHER_CONNECTIONS = (
+    "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
+    "WHERE datname = current_database() AND pid <> pg_backend_pid();"
+)
+
+
+def end_other_connections(*, db_url: str) -> None:
+    _run_reporting(["psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1",
+                    "-c", _END_OTHER_CONNECTIONS], 60)
+
+
+def wipe_db(*, db_url: str) -> None:
+    """Drop and recreate the public schema: an empty database, ready for a
+    dump or for `alembic upgrade head`."""
+    _run_reporting(
+        [
+            "psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1", "-c",
+            f"{_END_OTHER_CONNECTIONS} DROP SCHEMA IF EXISTS public CASCADE; "
+            "CREATE SCHEMA public;",
+        ],
+        120,
+        # Set for the session, not as a SET inside the same -c string.
+        env={**os.environ, "PGOPTIONS": f"-c lock_timeout={_RESTORE_LOCK_TIMEOUT}"},
+    )
+
+
+# A pg_restore newer than the server sets parameters the server doesn't have
+# (pg_restore 17 against Postgres 16: `SET transaction_timeout = 0`). The
+# statement fails and nothing depends on it, so its error is not a failed
+# restore.
+_HARMLESS_RESTORE_ERRORS = (
+    'unrecognized configuration parameter "transaction_timeout"',
+)
+
+
+def _restore_errors(stderr: str) -> list[str]:
+    """pg_restore's error lines, less the harmless ones."""
+    return [
+        line.strip() for line in stderr.splitlines()
+        if "error:" in line and not any(h in line for h in _HARMLESS_RESTORE_ERRORS)
+    ]
+
+
 def restore_db(*, db_url: str, sql_path: Path) -> None:
-    """Wipe public schema, then load from the custom-format dump. Raises on
-    subprocess failure; partial-restore state is the caller's concern."""
-    libpq = _libpq_url(db_url)
-    subprocess.run(
-        [
-            "psql", libpq, "-c",
-            "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;",
-        ],
-        capture_output=True, check=True, timeout=120,
-    )
-    # Custom-format (-Fc) dumps are restored with pg_restore, not psql.
-    subprocess.run(
-        ["pg_restore", "--no-owner", "--no-acl", "-d", libpq, str(sql_path)],
-        capture_output=True, check=True,
-        timeout=_DB_SUBPROCESS_TIMEOUT_S,
-    )
-
-
-def restore_images(*, images_root: Path, tar_path: Path) -> None:
-    """Untar over images_root.parent. Additive — files NOT in the
-    tarball are NOT removed. Caller wipes first if a clean restore
-    is needed."""
-    subprocess.run(
-        [
-            "tar", "--zstd", "-xf", str(tar_path),
-            "-C", str(images_root.parent),
-        ],
-        capture_output=True, check=True,
-        timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
-    )
+    """Wipe the public schema, then load the custom-format dump. Any error
+    pg_restore reports, other than a harmless one, raises RestoreFailed with
+    the first; the database is then partly loaded, and the caller decides what
+    to put back."""
+    wipe_db(db_url=db_url)
+    # Custom-format (-Fc) dumps are restored with pg_restore, not psql. It runs
+    # on past errors (it can't skip the harmless one and stop at the rest), so
+    # its errors are judged once it is done.
+    cmd = _gently(["pg_restore", "--no-owner", "--no-acl",
+                   "-d", _libpq_url(db_url), str(sql_path)])
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err,
+                              timeout=_RESTORE_SUBPROCESS_TIMEOUT_S)
+        if proc.returncode == 0:
+            return
+        err.seek(0)
+        said = err.read().decode(errors="replace")
+    errors = _restore_errors(said)
+    if errors:
+        raise RestoreFailed(errors[0] if len(errors) == 1
+                            else f"{errors[0]} ({len(errors)} errors in all)")
+    if not any(h in said for h in _HARMLESS_RESTORE_ERRORS):
+        # A failure with no error line to show for it.
+        lines = said.strip().splitlines()
+        raise RestoreFailed(lines[-1].strip() if lines
+                            else f"pg_restore exited with {proc.returncode}")
 
 
 def unlink_artifact_files(
     *,
     sql_path: str | None,
-    tar_path: str | None,
     manifest_path: str | None,
 ) -> dict:
     """Best-effort unlink of all on-disk files for a BackupRun row.
@@ -258,7 +433,6 @@ def unlink_artifact_files(
     deleted: dict = {}
     for label, p in (
         ("sql", sql_path),
-        ("tar", tar_path),
         ("manifest", manifest_path),
     ):
         if not p:

backend/app/services/backup_verify.py

@@ -0,0 +1,162 @@
+"""Testing a database dump without touching this install.
+
+The dump is loaded into a separate, temporary database on the same Postgres
+server, using the same steps a real restore takes. Every table is counted
+there and compared, the copy is brought up to this version, and the copy is
+then dropped. The live database is only read, to show its counts alongside.
+
+Every dump records how many rows each table held, counted in the same
+snapshot pg_dump dumped from, so the comparison is exact. For a dump in the
+bucket the counts sit beside it as `<dump name>.counts.json`; for a dump on
+this server they are in its manifest file. A dump taken before counts were
+recorded can still be tested: it is then compared with the live database,
+where a difference is what changed after the dump was taken, not a fault.
+
+The temporary database needs about as much free space on the database's disk
+as the live one takes, and only for the length of the test.
+"""
+from __future__ import annotations
+
+import json
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+from sqlalchemy.engine import make_url
+
+from . import backup_restore, backup_service
+from .backup_s3 import ObjectMissing, S3Target, get_file, put_file
+
+SCRATCH_SUFFIX = "_restore_test"
+
+
+def counts_key(dump_key: str) -> str:
+    """Where a dump's row counts sit in the bucket, beside the dump."""
+    return dump_key.removesuffix(".dump") + ".counts.json"
+
+
+def store_counts(target: S3Target, dump_key: str, counts: dict) -> None:
+    with tempfile.TemporaryDirectory() as d:
+        path = Path(d) / "counts.json"
+        path.write_text(json.dumps(counts))
+        put_file(target, counts_key(dump_key), path, content_type="application/json")
+
+
+def fetch_counts(target: S3Target, dump_key: str) -> dict | None:
+    """The counts recorded with a dump in the bucket, or None for a dump that
+    has none."""
+    with tempfile.TemporaryDirectory() as d:
+        path = Path(d) / "counts.json"
+        try:
+            get_file(target, counts_key(dump_key), path)
+        except ObjectMissing:
+            return None
+        return json.loads(path.read_text())
+
+
+def local_counts(manifest_path: str | None) -> dict | None:
+    """The counts in a dump's manifest file on this server, or None."""
+    try:
+        return json.loads(Path(manifest_path).read_text()).get("counts")
+    except (TypeError, OSError, ValueError):
+        return None
+
+
+def scratch_name(db_url: str) -> str:
+    """The temporary database's name: the live one's, plus SCRATCH_SUFFIX."""
+    return f"{make_url(db_url).database}{SCRATCH_SUFFIX}"
+
+
+def scratch_url(db_url: str) -> str:
+    return make_url(db_url).set(database=scratch_name(db_url)).render_as_string(
+        hide_password=False)
+
+
+def _on_server(db_url: str, statement) -> None:
+    """Run a CREATE or DROP DATABASE, which can't run inside a transaction."""
+    import psycopg
+
+    with psycopg.connect(backup_service._libpq_url(db_url), autocommit=True) as conn:
+        conn.execute(statement)
+
+
+def drop_scratch(db_url: str) -> None:
+    """Drop the temporary database if it is there, ending any connection to it."""
+    from psycopg import sql
+
+    _on_server(db_url, sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(
+        sql.Identifier(scratch_name(db_url))))
+
+
+def _create_scratch(db_url: str) -> None:
+    from psycopg import sql
+
+    _on_server(db_url, sql.SQL("CREATE DATABASE {} TEMPLATE template0").format(
+        sql.Identifier(scratch_name(db_url))))
+
+
+def trial_restore(*, db_url: str, dump_path: Path, recorded: dict | None,
+              note: Callable[[str], None] = lambda phase: None) -> dict:
+    """Load the dump at `dump_path` into the temporary database, compare it,
+    bring it up to this version, and drop it. `note` is told each phase as it
+    starts: "loading", "comparing", "upgrading". Raises RestoreFailed (or
+    S3Error from the upgrade) when the dump doesn't load or can't be brought
+    up; the temporary database is dropped either way."""
+    scratch = scratch_url(db_url)
+    if make_url(scratch).database == make_url(db_url).database:
+        raise ValueError("The test database would be the live one.")
+    drop_scratch(db_url)          # one left by a test cut off by a restart
+    _create_scratch(db_url)
+    try:
+        note("loading")
+        backup_service.restore_db(db_url=scratch, sql_path=dump_path)
+        note("comparing")
+        restored = backup_service.read_counts(scratch)
+        live = backup_service.read_counts(db_url)
+        note("upgrading")
+        backup_restore.migrate(db_name=scratch_name(db_url))
+        upgraded = backup_service.read_counts(scratch)["alembic"]
+    finally:
+        drop_scratch(db_url)
+    return {**compare(recorded, restored, live),
+            "alembic_dump": restored["alembic"], "alembic_now": upgraded}
+
+
+def compare(recorded: dict | None, restored: dict, live: dict) -> dict:
+    """Each table's rows in the dump (as recorded), in the restored copy, and
+    in the live database. With counts recorded, every table must come back
+    with exactly its count; `mismatched` names those that didn't."""
+    rec = (recorded or {}).get("tables")
+    names = sorted(set(restored["tables"]) | set(rec or {}) | set(live["tables"]))
+    tables = [
+        {"table": n, "dump": rec.get(n) if rec is not None else None,
+         "restored": restored["tables"].get(n), "live": live["tables"].get(n)}
+        for n in names
+    ]
+    mismatched = [t["table"] for t in tables
+                  if rec is not None and t["table"] in rec and t["restored"] != t["dump"]]
+    return {
+        "counted": rec is not None,
+        "tables": tables,
+        "mismatched": mismatched,
+        "rows_restored": sum(v for v in restored["tables"].values()),
+    }
+
+
+def summary(result: dict) -> str:
+    """One sentence on how a test went."""
+    n = len([t for t in result["tables"] if t["restored"] is not None])
+    rows = f"{result['rows_restored']:,}"
+    if result["mismatched"]:
+        shown = ", ".join(
+            f"{t['table']} ({t['dump']:,} in the dump, "
+            + ("table missing)" if t["restored"] is None else f"{t['restored']:,} restored)")
+            for t in result["tables"] if t["table"] in result["mismatched"][:5]
+        )
+        more = len(result["mismatched"]) - 5
+        return (f"{len(result['mismatched'])} table(s) came back different: {shown}"
+                + (f", and {more} more" if more > 0 else "") + ".")
+    if result["counted"]:
+        return f"All {n} tables came back with the {rows} rows the dump recorded."
+    return (f"The dump loaded cleanly: {n} tables, {rows} rows. It was taken before "
+            "dumps recorded their counts, so it is shown beside the live database instead.")

backend/app/services/cleanup_service.py

@@ -1044,45 +1044,75 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None:
 _ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"}
 
 
+def _stage_archive(archive_path: Path, staged: Path) -> None:
+    """Put the stored archive at `staged` for the importer to read: a symlink,
+    so a multi-gigabyte archive isn't copied, or a copy where links aren't
+    possible."""
+    import shutil
+
+    staged.unlink(missing_ok=True)
+    try:
+        staged.symlink_to(archive_path)
+    except OSError:
+        shutil.copy2(archive_path, staged)
+
+
+def _archive_folder_name(original_filename: str | None, attachment_id: int) -> str:
+    """The library folder a post-less archive's members land in: the archive's
+    own name without its extension, made safe for a path."""
+    stem = Path(original_filename or "").stem.strip().strip(".")
+    stem = "".join(c if c not in '/\\:*?"<>|' else "_" for c in stem)[:120].strip()
+    return stem or f"archive-{attachment_id}"
+
+
 def _reextract_archive_to_post(
     importer, archive_path: Path, post, source_row, artist, images_root: Path,
+    *, folder: str = "",
 ) -> list[int]:
-    """Extract one stored archive and link its members to `post`.
+    """Extract one stored archive and import its members.
 
     The stored attachment has no adjacent sidecar (it lives in the sha-addressed
-    attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's
-    library dir (`images_root/<slug>/<platform>/<post>/`) — the importer
-    re-derives the artist from the path AND copies members relative to it, so the
-    members land in the real library and resolve to the right artist — then re-run
-    `attach_in_place`: the archive extracts and `find_or_create_post` re-attaches
-    the members to the SAME Post (source_id + external_post_id). Removes only the
-    staged archive + sidecar afterward; the imported member files stay. Returns
-    the new member image ids.
+    attachment store), so the archive is staged under the artist's library dir
+    and `attach_in_place` re-run on it: the importer derives the artist from the
+    path and copies members relative to it, so they land in the real library.
+
+    - With a `post`, the stage is `images_root/<slug>/<platform>/<post>/` beside
+      a reconstructed sidecar, so `find_or_create_post` re-attaches the members
+      to that same Post (source_id + external_post_id).
+    - Without one (a filesystem import that had no sidecar), the stage is
+      `images_root/<slug>/<folder>/` with no sidecar, and the members belong to
+      the artist alone, as a filesystem import of the archive would have left
+      them.
+
+    Only the staged archive and sidecar are removed afterward; the imported
+    member files stay. Returns the new member image ids.
     """
     import json
-    import shutil
 
     from .archive_extractor import detect_archive_format
 
     fmt = detect_archive_format(archive_path)
     ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip")
-    platform = source_row.platform if source_row is not None else "imported"
-    sidecar = {
-        "category": source_row.platform if source_row is not None
-        else (post.raw_metadata or {}).get("category"),
-        "id": post.external_post_id,
-        "title": post.post_title or "",
-        "content": post.description or "",
-        "published_at": post.post_date.isoformat() if post.post_date else None,
-        "url": post.post_url,
-    }
-    work = images_root / artist.slug / platform / str(post.external_post_id)
+    if post is not None:
+        platform = source_row.platform if source_row is not None else "imported"
+        work = images_root / artist.slug / platform / str(post.external_post_id)
+    else:
+        work = images_root / artist.slug / folder
     work.mkdir(parents=True, exist_ok=True)
     staged = work / f"archive{ext}"  # clean ext → is_archive + find_sidecar
     sidecar_path = staged.with_suffix(".json")
     try:
-        shutil.copy2(archive_path, staged)
-        sidecar_path.write_text(json.dumps(sidecar))
+        _stage_archive(archive_path, staged)
+        if post is not None:
+            sidecar_path.write_text(json.dumps({
+                "category": source_row.platform if source_row is not None
+                else (post.raw_metadata or {}).get("category"),
+                "id": post.external_post_id,
+                "title": post.post_title or "",
+                "content": post.description or "",
+                "published_at": post.post_date.isoformat() if post.post_date else None,
+                "url": post.post_url,
+            }))
         res = importer.attach_in_place(staged, artist=artist, source=source_row)
         return list(res.member_image_ids or [])
     finally:
@@ -1124,8 +1154,9 @@ def reextract_archive_attachments(
 
     summary = {
         "scanned": 0, "archives": 0, "members_imported": 0,
-        "posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0,
-        "errors": 0, "partial": False, "resume_after_id": after_id,
+        "posts_touched": 0, "without_post": 0, "already_extracted": 0,
+        "skipped_no_artist": 0, "errors": 0, "partial": False,
+        "resume_after_id": after_id,
     }
     settings = ImportSettings.load_sync(session)
     importer = Importer(
@@ -1150,15 +1181,17 @@ def reextract_archive_attachments(
         except OSError:
             continue
         summary["archives"] += 1
-        if att.post_id is None:
-            summary["skipped_no_post"] += 1
-            continue
-        post = session.get(Post, att.post_id)
-        if post is None:
-            summary["skipped_no_post"] += 1
+        # Members already recorded as coming from this archive: done before.
+        if session.execute(
+            select(ImageProvenance.id)
+            .where(ImageProvenance.from_attachment_id == att.id)
+            .limit(1)
+        ).first() is not None:
+            summary["already_extracted"] += 1
             continue
+        post = session.get(Post, att.post_id) if att.post_id else None
         artist = session.get(Artist, att.artist_id) if att.artist_id else None
-        if artist is None and post.artist_id:
+        if artist is None and post is not None and post.artist_id:
             artist = session.get(Artist, post.artist_id)
         if artist is None or not artist.slug:
             # The importer re-derives the artist from the staged path, so we need
@@ -1166,20 +1199,35 @@ def reextract_archive_attachments(
             # subscription posts; skip rather than orphan the members.)
             summary["skipped_no_artist"] += 1
             continue
-        source_row = session.get(Source, post.source_id) if post.source_id else None
+        source_row = (
+            session.get(Source, post.source_id)
+            if post is not None and post.source_id else None
+        )
+        if post is None:
+            summary["without_post"] += 1
+        # Logged on the way in as well as out, so an archive that stalls a
+        # chunk is named in the log even if the worker dies holding it.
+        size_mb = (att.size_bytes or 0) / 1e6
+        log.info("re-extract: attachment %s (%.0f MB) starting", att.id, size_mb)
+        archive_start = time.monotonic()
         try:
             ids = _reextract_archive_to_post(
                 importer, stored, post, source_row, artist, images_root,
+                folder=_archive_folder_name(att.original_filename, att.id),
             )
             session.commit()
         except Exception as exc:  # one bad archive must not strand the rest
             session.rollback()
             summary["errors"] += 1
-            log.warning("re-extract failed for attachment %s: %s", att.id, exc)
+            log.warning("re-extract failed for attachment %s after %.1fs: %s",
+                        att.id, time.monotonic() - archive_start, exc)
             continue
+        log.info("re-extract: attachment %s done in %.1fs, %d new image(s)",
+                 att.id, time.monotonic() - archive_start, len(ids))
         if ids:
             summary["members_imported"] += len(ids)
-            summary["posts_touched"] += 1
+            if post is not None:
+                summary["posts_touched"] += 1
             enqueue_ids.extend(ids)
 
         # Time-box the chunk. resume_after_id already points at this attachment,

backend/app/services/db_helpers.py

@@ -21,7 +21,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..models import Source
-from .gallery_dl import ErrorType
+from .download_result import ErrorType
 
 
 async def get_or_create[T](

backend/app/services/discord_client.py

@@ -0,0 +1,729 @@
+"""Native Discord read client — the Discord counterpart to subscribestar_client.
+
+Mirrors gallery-dl 1.32.13's `extractor/discord.py` (rule 130: gallery-dl is the
+known-working base), adapted to the native core's client contract
+(`ingest_core` module docstring): `iter_posts` / `extract_media`, plus the
+post-first `post_record_key` and the `post_meta` date the revisit window reads.
+
+What is mirrored exactly, because drift in any of it changes what we fetch or
+where it lands on disk:
+
+  - API v10, `Authorization: <user token>` (a USER token, not a bot token).
+  - gallery-dl's request profile: its date-derived Firefox User-Agent,
+    `Accept: */*`, `Accept-Language`, `Referer: https://discord.com/`.
+  - `GET /channels/{id}/messages?limit=100&before=<last id>`, newest first,
+    stopping on a short page. Message types {0, 19, 21} only.
+  - The walk: a text/news channel's own messages then its threads, a forum's
+    threads, a category's children, a server's text/news/forum channels.
+  - Files: attachments, then embeds of type image/gifv/video (FC configures
+    `embeds: all`, which for files is the same three plus rich/link embeds that
+    carry an image), then forwarded `message_snapshots`, numbered from 1 across
+    the lot — the `num` in `{date}_{message_id}_{num}_{filename}`.
+  - Text: `content`, rich-embed author/title/description/fields/footer, poll.
+
+Two deliberate departures, both about the walk order, neither about content:
+
+  - Threads are walked newest-CREATED first (by id), not by last-message time.
+    A backfill resumes from a checkpointed channel; last-message order shifts
+    between chunks whenever someone posts, which can move an unwalked thread
+    above the resume point and skip it. Creation order only ever grows at the
+    front, where the next tick finds it.
+  - A 403 on a thread or a nested channel skips that feed instead of failing
+    the walk. gallery-dl skips only nested channels; one private thread the
+    token cannot read would otherwise stop every channel after it.
+
+And one about content: a message is taken only if it has an image or video
+ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
+as post, and the operator wants the art, not the conversation (2026-09-28): a
+text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
+chat. A message that passes keeps every file gallery-dl would take, numbered
+as gallery-dl numbers them, so the on-disk names still match.
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+import time
+from collections.abc import Iterator
+from datetime import date
+from urllib.parse import unquote
+
+import requests
+
+from .native_ingest_common import (
+    MediaItem,
+    NativeAuthError,
+    NativeDriftError,
+    NativeIngestError,
+    retry_after_seconds,
+)
+
+log = logging.getLogger(__name__)
+
+API_ROOT = "https://discord.com/api/v10"
+_ROOT = "https://discord.com"
+
+_TIMEOUT_SECONDS = 60.0
+_MESSAGES_BATCH = 100
+_THREADS_BATCH = 25
+# gallery-dl retries a 429 up to its default 4 retries, waiting
+# `request_interval_429` (60s) between them. Discord's Retry-After is exact, so
+# it is honoured when present; 60s is the fallback and the cap.
+_MAX_429_RETRIES = 4
+_429_WAIT_SECONDS = 60.0
+# How far back the poster picker looks: two pages. Enough to see who posts in
+# a channel, few enough to answer while the operator waits (#4491).
+_POSTER_SCAN = 200
+
+# https://discord.com/developers/docs/resources/message#message-object-message-types
+# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
+MESSAGE_TYPES = frozenset({0, 19, 21})
+# https://discord.com/developers/docs/resources/channel#channel-object-channel-types
+_TEXT = frozenset({0, 5})               # text, announcement: messages + threads
+_DIRECT = frozenset({1, 3, 10, 11, 12})  # DMs and threads: messages only
+_FORUM = frozenset({15, 16})            # forum, media: threads only
+_CATEGORY = 4
+_SERVER_WALK = _TEXT | _FORUM
+_EMBED_TYPES = frozenset({"image", "gifv", "video"})
+# What makes a message art rather than chat: an attached file FC imports as an
+# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
+# import pipeline, and this one is a network client).
+_VISUAL_EXTS = frozenset({
+    "png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
+    "mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
+})
+
+_URL_RE = re.compile(
+    r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
+    r"(?P<server>@me|\d+)(?:/(?:\d+/threads/)?(?P<channel>\d+))?(?P<rest>/.*)?/?$"
+)
+
+
+class DiscordAPIError(NativeIngestError):
+    """Base for native Discord client failures."""
+
+
+class DiscordAuthError(DiscordAPIError, NativeAuthError):
+    """401 (the token is invalid or expired) or a 403 on the channel the
+    source names. The fix is a new token, not a new client."""
+
+
+class DiscordDriftError(DiscordAPIError, NativeDriftError):
+    """A response did not have the shape the walk depends on."""
+
+
+def firefox_user_agent(today: date | None = None) -> str:
+    """gallery-dl's default User-Agent: a Firefox whose version advances every
+    four weeks (`util._ff_ver`, "147 on 2026-01-13"). Computed the same way so
+    the profile keeps matching the gallery-dl this replaced."""
+    ver = ((today or date.today()).toordinal() - 735_513) // 28
+    return (
+        f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:{ver}.0) "
+        f"Gecko/20100101 Firefox/{ver}.0"
+    )
+
+
+def nameext_from_url(url: str) -> tuple[str, str]:
+    """gallery-dl's `text.nameext_from_url`: the URL's last path segment,
+    unquoted, split at the last dot when the extension is at most 16 chars
+    (lowercased); otherwise the whole name and no extension."""
+    filename = unquote(url.partition("?")[0].rpartition("/")[2])
+    name, _, ext = filename.rpartition(".")
+    if name and len(ext) <= 16:
+        return name, ext.lower()
+    return filename, ""
+
+
+def parse_source_url(url: str) -> tuple[str | None, str | None]:
+    """`(server_id, channel_id)` from a Discord channel/server URL. `server_id`
+    is None for a DM (`@me`); `channel_id` is None for a whole server. Raises
+    DiscordAPIError for anything else, including a link to a single message —
+    a message is not something a source can subscribe to."""
+    m = _URL_RE.match((url or "").strip())
+    if not m or (m.group("rest") or "").strip("/"):
+        raise DiscordAPIError(
+            f"Not a Discord channel or server link: {url!r} "
+            "(expected https://discord.com/channels/<server>[/<channel>])"
+        )
+    server = m.group("server")
+    channel = m.group("channel")
+    if server == "@me":
+        if not channel:
+            raise DiscordAPIError(f"A DM link needs a channel id: {url!r}")
+        return None, channel
+    return server, channel
+
+
+def message_text(message: dict) -> str:
+    """gallery-dl's `extract_message_text`: the body plus the text of rich
+    embeds and polls, newline-joined, empties dropped."""
+    parts = [message.get("content") or ""]
+    for embed in message.get("embeds") or []:
+        if embed.get("type") != "rich":
+            continue
+        parts.append((embed.get("author") or {}).get("name") or "")
+        parts.append(embed.get("title") or "")
+        parts.append(embed.get("description") or "")
+        for fld in embed.get("fields") or []:
+            parts.append(fld.get("name") or "")
+            parts.append(fld.get("value") or "")
+        parts.append((embed.get("footer") or {}).get("text") or "")
+    poll = message.get("poll")
+    if poll:
+        parts.append(((poll.get("question") or {}).get("text")) or "")
+        for answer in poll.get("answers") or []:
+            parts.append(((answer.get("poll_media") or {}).get("text")) or "")
+    return "\n".join(p for p in parts if p)
+
+
+def _message_and_snapshots(message: dict) -> list[dict]:
+    """The message itself, then each forwarded message it carries."""
+    return [message] + [
+        (s or {}).get("message") or {}
+        for s in message.get("message_snapshots") or []
+        if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
+    ]
+
+
+def _is_visual(attachment: dict) -> bool:
+    ctype = (attachment.get("content_type") or "").lower()
+    if ctype.startswith(("image/", "video/")):
+        return True
+    name = attachment.get("filename") or attachment.get("url") or ""
+    return nameext_from_url(name)[1] in _VISUAL_EXTS
+
+
+def has_visual_attachment(message: dict) -> bool:
+    """Does this message, or a message it forwards, have an image or video
+    attached? The one test for "art, not chat" — see the module docstring."""
+    return any(
+        att.get("url") and _is_visual(att)
+        for snap in _message_and_snapshots(message)
+        for att in snap.get("attachments") or []
+    )
+
+
+_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
+# Words a server name wraps around its creator's: "Todding's Server",
+# "The Official Todding Discord".
+_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
+
+
+def _squash(text: str | None) -> str:
+    return _SQUASH.sub("", (text or "").lower())
+
+
+def _server_tokens(server_name: str | None) -> list[str]:
+    name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
+    words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
+    tokens = [w for w in words if len(w) >= 3]
+    joined = "".join(words)
+    if len(joined) >= 3 and joined not in tokens:
+        tokens.append(joined)
+    return tokens
+
+
+def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
+    """Mark who is probably the creator a server is named for, and order the
+    list for picking: suggested first, then by images posted, then messages.
+
+    Two signals, each a reason the picker shows beside the name:
+      - they own the server;
+      - their username or display name matches the server's name, once
+        "'s Server", "Official", "Discord" and punctuation are taken off
+        ("Todding's Server" and "todding").
+    Neither signal alone is proof, which is why this suggests and never
+    decides: the operator ticks the list. Bots are never suggested."""
+    tokens = _server_tokens(server_name)
+    owner = str(owner_id) if owner_id else None
+    ranked = []
+    for p in posters:
+        reasons = []
+        if owner and p.get("id") == owner:
+            reasons.append("owns the server")
+        names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
+        if any(t in n or n in t for t in tokens for n in names):
+            reasons.append("name matches the server")
+        ranked.append({**p, "reasons": reasons, "suggested": False})
+    best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
+    for r in ranked:
+        r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
+    ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
+    return ranked
+
+
+class DiscordClient:
+    """Synchronous Discord API v10 read client for one user token."""
+
+    def __init__(
+        self,
+        token: str | None,
+        *,
+        request_sleep: float = 0.0,
+        max_retries: int = _MAX_429_RETRIES,
+        session: requests.Session | None = None,
+    ):
+        self._session = session or requests.Session()
+        self._session.headers.update({
+            "User-Agent": firefox_user_agent(),
+            "Accept": "*/*",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Referer": _ROOT + "/",
+        })
+        if token:
+            self._session.headers["Authorization"] = token
+        self._token = token
+        self._request_sleep = request_sleep or 0.0
+        self._max_retries = max_retries
+        self._server: dict = {}
+        self._channels: dict[str, dict] = {}
+        self._skip_feed = False
+        # Whose messages the walk takes (`only_from`); empty takes everyone's.
+        self._authors: frozenset[str] = frozenset()
+
+    # -- request -----------------------------------------------------------
+
+    def _get(self, endpoint: str, params: dict | None = None):
+        if not self._token:
+            raise DiscordAuthError("No Discord token is configured for this source")
+        if self._request_sleep > 0:
+            time.sleep(self._request_sleep)
+        url = API_ROOT + endpoint
+        attempt = 0
+        while True:
+            try:
+                resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
+            except requests.RequestException as exc:
+                raise DiscordAPIError(f"Discord request failed ({endpoint}): {exc}") from exc
+            if resp.status_code == 429 and attempt < self._max_retries:
+                attempt += 1
+                delay = retry_after_seconds(
+                    resp, attempt, base=_429_WAIT_SECONDS, cap=_429_WAIT_SECONDS,
+                )
+                log.warning(
+                    "Discord 429 (%s) — waiting %.1fs (retry %d/%d)",
+                    endpoint, delay, attempt, self._max_retries,
+                )
+                time.sleep(delay)
+                continue
+            break
+        if resp.status_code == 401:
+            raise DiscordAuthError(
+                "Discord rejected the token (HTTP 401) — it is invalid or has "
+                "expired; copy a fresh one from the browser",
+                status_code=401,
+            )
+        if resp.status_code != 200:
+            raise DiscordAPIError(
+                f"Discord returned HTTP {resp.status_code} ({endpoint})",
+                status_code=resp.status_code,
+                retry_after=_retry_after(resp),
+            )
+        try:
+            return resp.json()
+        except ValueError as exc:
+            raise DiscordDriftError(
+                f"Discord returned non-JSON for {endpoint} ({len(resp.content)} bytes)"
+            ) from exc
+
+    # -- metadata (gallery-dl parse_server / parse_channel) -----------------
+
+    def _load_server(self, server_id: str) -> None:
+        server = self._get(f"/guilds/{server_id}")
+        if not isinstance(server, dict) or "id" not in server:
+            raise DiscordDriftError(f"Discord server {server_id} came back without an id")
+        self._server = {
+            "server": server.get("name") or "",
+            "server_id": str(server["id"]),
+            "owner_id": server.get("owner_id"),
+        }
+        channels = self._get(f"/guilds/{server_id}/channels")
+        if not isinstance(channels, list):
+            raise DiscordDriftError(f"Discord server {server_id} channel list is not a list")
+        # Categories first, so every child can name its parent.
+        for channel in sorted(channels, key=lambda ch: ch.get("type") != _CATEGORY):
+            self._parse_channel(channel)
+
+    def _parse_channel(self, channel: dict) -> dict:
+        parent_id = channel.get("parent_id")
+        meta = {
+            "channel": channel.get("name") or "",
+            "channel_id": str(channel.get("id")),
+            "channel_type": channel.get("type"),
+            "channel_topic": channel.get("topic") or "",
+            "parent_id": parent_id,
+            "is_thread": "thread_metadata" in channel,
+        }
+        parent = self._channels.get(parent_id) if parent_id else None
+        if parent:
+            meta["parent"] = parent["channel"]
+            meta["parent_type"] = parent["channel_type"]
+        if meta["channel_type"] in {1, 3}:
+            recipients = channel.get("recipients") or []
+            meta["channel"] = "DMs"
+            meta["recipients"] = [u.get("username") for u in recipients]
+            meta["recipients_id"] = [u.get("id") for u in recipients]
+        self._channels[meta["channel_id"]] = meta
+        return meta
+
+    def _channel_meta(self, channel_id: str) -> dict:
+        if channel_id not in self._channels:
+            self._parse_channel(self._get(f"/channels/{channel_id}"))
+        return self._channels[channel_id]
+
+    def _threads(self, channel_id: str) -> list[dict]:
+        """Every thread of a channel or forum, newest-created first (see the
+        module docstring for why not last-message order)."""
+        threads: list[dict] = []
+        offset = 0
+        while True:
+            data = self._get(f"/channels/{channel_id}/threads/search", {
+                "sort_by": "last_message_time",
+                "sort_order": "desc",
+                "limit": _THREADS_BATCH,
+                "offset": offset,
+            })
+            batch = (data.get("threads") or []) if isinstance(data, dict) else []
+            threads.extend(batch)
+            if len(batch) < _THREADS_BATCH:
+                break
+            offset += len(batch)
+        threads.sort(key=lambda t: int(t.get("id") or 0), reverse=True)
+        return threads
+
+    # -- the walk ------------------------------------------------------------
+
+    def _feeds(self, channel_id: str, *, safe: bool) -> Iterator[tuple[str, bool]]:
+        """`(channel_id, safe)` for every message feed under `channel_id`, in
+        gallery-dl's order. `safe` feeds are skipped on a 403."""
+        try:
+            ctype = self._channel_meta(channel_id)["channel_type"]
+        except DiscordAPIError as exc:
+            if exc.status_code != 403:
+                raise
+            if not safe:
+                raise DiscordAuthError(
+                    f"The Discord token cannot see channel {channel_id} (HTTP 403)",
+                    status_code=403,
+                ) from exc
+            log.info("Discord: no access to channel %s — skipped", channel_id)
+            return
+        if ctype in _TEXT or ctype in _DIRECT:
+            yield channel_id, safe
+        if ctype in _TEXT or ctype in _FORUM:
+            try:
+                threads = self._threads(channel_id)
+            except DiscordAPIError as exc:
+                if exc.status_code != 403:
+                    raise
+                log.info("Discord: cannot list threads of %s — skipped", channel_id)
+                threads = []
+            for thread in threads:
+                yield self._parse_channel(thread)["channel_id"], True
+        elif ctype == _CATEGORY:
+            for child in list(self._channels.values()):
+                if child.get("parent_id") == channel_id:
+                    yield from self._feeds(child["channel_id"], safe=True)
+        elif ctype not in _DIRECT and not safe:
+            raise DiscordAPIError(
+                f"Discord channel {channel_id} is of type {ctype}, which has no messages"
+            )
+
+    def _source_feeds(self, url: str) -> Iterator[tuple[str, bool]]:
+        server_id, channel_id = parse_source_url(url)
+        self._server, self._channels = {}, {}
+        if server_id is not None:
+            self._load_server(server_id)
+        if channel_id is not None:
+            yield from self._feeds(channel_id, safe=False)
+            return
+        for meta in list(self._channels.values()):
+            if meta["channel_type"] in _SERVER_WALK:
+                yield from self._feeds(meta["channel_id"], safe=True)
+
+    def only_from(self, authors) -> None:
+        """Take only messages posted by these people: user ids, usernames or
+        display names, any case. A creator's server is full of other members
+        posting their own pictures; the source is subscribed to the creator.
+        Empty or None takes everyone's, as before."""
+        self._authors = frozenset(
+            str(a).strip().lower() for a in authors or () if str(a).strip()
+        )
+
+    def _author_wanted(self, message: dict) -> bool:
+        if not self._authors:
+            return True
+        author = message.get("author") or {}
+        names = (author.get("id"), author.get("username"), author.get("global_name"))
+        return any(str(n).lower() in self._authors for n in names if n)
+
+    def source_label(self, url: str) -> str | None:
+        """What the source is called in Discord — `Server · #channel`, a
+        thread as `#parent › thread`, a whole server by its name — from the
+        metadata the walk loaded. None when the walk never got that far."""
+        try:
+            _, channel_id = parse_source_url(url)
+        except DiscordAPIError:
+            return None
+        server = self._server.get("server") or None
+        if channel_id is None:
+            return server
+        meta = self._channels.get(channel_id) or {}
+        name = meta.get("channel")
+        if not name:
+            return None
+        if name != "DMs":
+            name = f"#{name}"
+            if meta.get("is_thread") and meta.get("parent"):
+                name = f"#{meta['parent']} › {meta['channel']}"
+        return f"{server} · {name}" if server else name
+
+    def skip_feed(self) -> None:
+        """Optional core seam (#4413): end the current channel and go on to the
+        next one. A tick's early-out means THIS channel has nothing new, not
+        that the server has nothing new."""
+        self._skip_feed = True
+
+    def iter_posts(
+        self, campaign_id: str, cursor: str | None = None
+    ) -> Iterator[tuple[dict, dict, str | None]]:
+        """Yield `(message, channel_meta, page_cursor)` for every content
+        message the source reaches, channel by channel, each newest first.
+
+        `campaign_id` is the source URL. The cursor is `<channel_id>:<before>`
+        — the channel and the `before` id that fetched the page (empty for a
+        channel's first page) — so a backfill resumes inside the right channel
+        and re-fetches the page it was cut in. A cursor naming a channel the
+        walk no longer reaches (a deleted thread) restarts from the top rather
+        than walking nothing.
+        """
+        resume_channel, _, resume_before = (cursor or "").partition(":")
+        resuming = bool(resume_channel)
+        feeds = list(self._source_feeds(campaign_id)) if resuming else None
+        if feeds is not None and resume_channel not in {cid for cid, _ in feeds}:
+            log.warning(
+                "Discord: resume channel %s is no longer in %s — restarting",
+                resume_channel, campaign_id,
+            )
+            resuming = False
+        for channel_id, safe in feeds if feeds is not None else self._source_feeds(campaign_id):
+            before = None
+            if resuming:
+                if channel_id != resume_channel:
+                    continue
+                resuming = False
+                before = resume_before or None
+            yield from self._iter_channel(channel_id, before, safe=safe)
+
+    def _iter_channel(
+        self, channel_id: str, before: str | None, *, safe: bool
+    ) -> Iterator[tuple[dict, dict, str | None]]:
+        self._skip_feed = False
+        meta = {**self._server, **self._channels.get(channel_id, {})}
+        while True:
+            page_cursor = f"{channel_id}:{before or ''}"
+            try:
+                messages = self._get(
+                    f"/channels/{channel_id}/messages",
+                    {"limit": _MESSAGES_BATCH, "before": before},
+                )
+            except DiscordAPIError as exc:
+                if exc.status_code != 403:
+                    raise
+                if not safe:
+                    raise DiscordAuthError(
+                        f"The Discord token cannot read channel {channel_id} (HTTP 403)",
+                        status_code=403,
+                    ) from exc
+                log.info("Discord: no access to messages of %s — skipped", channel_id)
+                return
+            if not isinstance(messages, list):
+                raise DiscordDriftError(
+                    f"Discord messages of {channel_id} came back as "
+                    f"{type(messages).__name__}, not a list"
+                )
+            for message in messages:
+                if message.get("type") not in MESSAGE_TYPES:
+                    continue
+                if not self._author_wanted(message):
+                    continue
+                message["_meta"] = meta
+                yield message, meta, page_cursor
+                if self._skip_feed:
+                    return
+            if len(messages) < _MESSAGES_BATCH:
+                return
+            before = str(messages[-1]["id"])
+
+    # -- per-message -------------------------------------------------------
+
+    @staticmethod
+    def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
+        """gallery-dl's file list for one message: attachments, then the first
+        of video/image/thumbnail `proxy_url` of each file-bearing embed, then
+        the same for every forwarded snapshot; numbered from 1 across them.
+
+        Empty for a message with no image or video attached: that is chat,
+        whatever else it carries (`has_visual_attachment`)."""
+        if not has_visual_attachment(post):
+            return []
+        mid = str(post.get("id") or "")
+        snapshots = _message_and_snapshots(post)
+        found: list[tuple[str, str, str | None]] = []
+        for snap in snapshots:
+            for att in snap.get("attachments") or []:
+                if att.get("url"):
+                    aid = att.get("id")
+                    found.append((att["url"], "attachment", str(aid) if aid else None))
+            for embed in snap.get("embeds") or []:
+                if embed.get("type") not in _EMBED_TYPES:
+                    continue
+                for fld in ("video", "image", "thumbnail"):
+                    url = (embed.get(fld) or {}).get("proxy_url")
+                    if url:
+                        found.append((url, "embed", None))
+                        break
+        items = []
+        for num, (url, kind, fid) in enumerate(found, start=1):
+            name, ext = nameext_from_url(url)
+            if fid is None:
+                path = url.partition("?")[0].encode()
+                fid = "u" + hashlib.sha1(path, usedforsecurity=False).hexdigest()[:32]
+            items.append(MediaItem(
+                url=url, filename=name, extension=ext, kind=kind, post_id=mid,
+                num=num, media_id=fid,
+            ))
+        return items
+
+    @staticmethod
+    def post_meta(post: dict) -> dict:
+        """No title (Discord has none); `date` is the message timestamp, ISO
+        with an offset — what the core's revisit window reads."""
+        return {"title": None, "date": post.get("timestamp")}
+
+    @classmethod
+    def post_record_key(cls, post: dict) -> tuple[str, str] | None:
+        """`(message:<id>, <id>)` — gates the message record through the seen
+        ledger, like `post:<id>` on the other platforms.
+
+        None for a message `extract_media` takes nothing from, i.e. one with
+        no image or video attached. The drop grouping (discord_grouping) is
+        built on that: a channel's chatter recorded as posts would bury the
+        drops it exists to surface."""
+        mid = post.get("id")
+        mid = str(mid) if mid is not None else ""
+        if not mid or not cls.extract_media(post):
+            return None
+        return (f"message:{mid}", mid)
+
+    # -- verify ------------------------------------------------------------
+
+    def describe(self, server_id: str | None, channel_id: str | None) -> dict:
+        """The display names behind a server/channel pair, for the browser
+        extension's Add panel. Best-effort per name: one that can't be read
+        comes back None, and the other is still returned."""
+        out: dict = {"server": None, "channel": None, "parent": None}
+        if server_id:
+            try:
+                out["server"] = (self._get(f"/guilds/{server_id}") or {}).get("name") or None
+            except DiscordAPIError:
+                pass
+        if channel_id:
+            try:
+                meta = self._parse_channel(self._get(f"/channels/{channel_id}"))
+                out["channel"] = meta.get("channel") or None
+            except (DiscordAPIError, AttributeError):
+                pass
+        return out
+
+    def recent_posters(
+        self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
+    ) -> dict:
+        """Who has posted lately in a channel, for picking a source's poster
+        list (#4491): the newest `max_messages` messages, tallied per author.
+        Deliberately shallow — it answers "who posts here", not "who ever did".
+        Every poster comes back with their stable id, which is what the list
+        stores: a username can change on a whim, an id can't."""
+        server: dict = {}
+        if server_id:
+            try:
+                server = self._get(f"/guilds/{server_id}") or {}
+            except DiscordAPIError:
+                server = {}
+        tally: dict[str, dict] = {}
+        before = None
+        seen = 0
+        while seen < max_messages:
+            page = self._get(
+                f"/channels/{channel_id}/messages",
+                {"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
+            )
+            if not isinstance(page, list) or not page:
+                break
+            for message in page:
+                seen += 1
+                if message.get("type") not in MESSAGE_TYPES:
+                    continue
+                author = message.get("author") or {}
+                aid = str(author.get("id") or "")
+                if not aid:
+                    continue
+                row = tally.setdefault(aid, {
+                    "id": aid,
+                    "username": author.get("username"),
+                    "global_name": author.get("global_name"),
+                    "bot": bool(author.get("bot")),
+                    "messages": 0,
+                    "images": 0,
+                })
+                row["messages"] += 1
+                if has_visual_attachment(message):
+                    row["images"] += 1
+            if len(page) < _MESSAGES_BATCH:
+                break
+            before = str(page[-1]["id"])
+        return {
+            "server": server.get("name") or None,
+            "owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
+            "scanned": seen,
+            "posters": rank_posters(
+                server.get("name"), server.get("owner_id"), list(tally.values()),
+            ),
+        }
+
+    def verify_auth(self, url: str) -> tuple[bool | None, str]:
+        """Is the token valid, and can it see what the source names?"""
+        try:
+            server_id, channel_id = parse_source_url(url)
+        except DiscordAPIError as exc:
+            return None, str(exc)
+        try:
+            me = self._get("/users/@me")
+            if channel_id is not None:
+                self._get(f"/channels/{channel_id}")
+            elif server_id is not None:
+                self._get(f"/guilds/{server_id}")
+        except DiscordAuthError as exc:
+            return False, f"Discord rejected the token — {exc}"
+        except DiscordAPIError as exc:
+            if exc.status_code in (403, 404):
+                return False, (
+                    "The token is valid, but its account cannot see "
+                    f"{'this channel' if channel_id else 'this server'} "
+                    f"(HTTP {exc.status_code})"
+                )
+            return None, f"Couldn't verify (network/HTTP issue): {exc}"
+        who = (me or {}).get("username") if isinstance(me, dict) else None
+        return True, f"Token valid{f' ({who})' if who else ''} — the source is readable."
+
+
+def _retry_after(resp: requests.Response) -> float | None:
+    hdr = resp.headers.get("Retry-After")
+    try:
+        return float(hdr) if hdr else None
+    except (TypeError, ValueError):
+        return None

backend/app/services/discord_downloader.py

@@ -0,0 +1,160 @@
+"""Native Discord media downloader — the Discord counterpart to
+subscribestar_downloader.
+
+Writes files exactly where gallery-dl wrote them, so a cutover finds every
+existing file on disk (`skipped_disk`) instead of fetching it again:
+
+    <images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>
+
+That is what FC's gallery-dl config produced (directory `{channel}`, filename
+`{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the
+per-source base directory `<images>/<artist>/<platform>`), retired from that
+config once Discord moved here; tests/test_discord_naming.py pins the match
+against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it
+on Linux — `/` becomes `_` and control characters are removed, nothing else
+(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
+whose Windows set would turn a `:` in a channel or file name into `_` and miss
+the file gallery-dl wrote.
+
+Post-first (rule 120): each file gets a minimal sidecar named like it minus the
+extension (what `find_sidecar` pairs first), and the message itself gets one
+record, `<YYYYMMDD>_<message_id>_post.json`, carrying gallery-dl's metadata keys
+— `message_id` for the post id, `server_id`/`channel_id` for the permalink,
+`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read
+the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both
+outrank `message_id` in the post-id chain (`platforms.base`).
+
+PURE: no DB; the seen-skip is an injected predicate.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+
+import requests
+
+from .discord_client import firefox_user_agent, message_text
+from .native_ingest_common import (
+    BaseNativeDownloader,
+    MediaItem,
+    PostRecordOutcome,
+    make_session,
+)
+
+PLATFORM = "discord"
+_CONTROL = re.compile("[\x00-\x1f\x7f]")
+# gallery-dl falls back to the response's type for a URL with no extension;
+# we never see the response before naming, and such URLs do not occur for
+# Discord attachments or embed proxies in practice.
+_NO_EXTENSION = "bin"
+
+
+def gdl_clean(segment: str) -> str:
+    """One path segment as gallery-dl writes it on Linux."""
+    return _CONTROL.sub("", segment.replace("/", "_"))
+
+
+def message_date(post: dict) -> datetime | None:
+    raw = post.get("timestamp")
+    if not isinstance(raw, str) or not raw:
+        return None
+    try:
+        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC)
+
+
+def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path:
+    """gallery-dl's `{channel}` directory; an empty name adds no segment."""
+    base = Path(images_root) / artist_slug / PLATFORM
+    channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip())
+    return base / channel if channel else base
+
+
+def media_stem(post: dict, media) -> str:
+    """`<YYYYMMDD>_<message_id>_<NN>_<name>` — the file's name minus `.<ext>`."""
+    when = message_date(post)
+    day = f"{when:%Y%m%d}" if when else "None"
+    return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}")
+
+
+class DiscordDownloader(BaseNativeDownloader):
+    """Download a message's files to gallery-dl's layout. The CDN gets
+    gallery-dl's browser profile and no token — gallery-dl sends the token only
+    to the API, and the CDN URLs are pre-signed."""
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None = None,
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        session: requests.Session | None = None,
+    ):
+        super().__init__(
+            images_root, None, platform=PLATFORM,
+            validate=validate, rate_limit=rate_limit,
+            session=session if session is not None else make_session(None, extra_headers={
+                "User-Agent": firefox_user_agent(),
+                "Accept-Language": "en-US,en;q=0.5",
+                "Referer": "https://discord.com/",
+            }),
+        )
+
+    # The download loop is BaseNativeDownloader's; these seams give it
+    # gallery-dl's Discord layout.
+
+    def _post_folder(self, post: dict, artist_slug: str) -> Path:
+        return channel_dir(self.images_root, artist_slug, post)
+
+    def _media_name(self, post: dict, media: MediaItem, index: int) -> str:
+        return f"{media_stem(post, media)}.{media.extension or _NO_EXTENSION}"
+
+    def _media_sidecar(self, post: dict, media: MediaItem) -> dict:
+        # `message_id`, never `id`: `id` outranks it in the post-id chain.
+        return {"category": PLATFORM, "message_id": str(post.get("id") or ""),
+                "source_url": media.url}
+
+    def write_post_record(
+        self, post: dict, artist_slug: str, *, revisit: bool = False,
+    ) -> PostRecordOutcome:
+        """The message record — the one writer of a Discord post's body, date
+        and permalink ids. `revisit` re-reads a message already captured (an
+        edit); an empty re-read writes nothing, so it never blanks a body."""
+        mid = str(post.get("id") or "")
+        body = message_text(post)
+        if not mid or (revisit and not body.strip()):
+            return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0)
+        meta = post.get("_meta") or {}
+        author = post.get("author") or {}
+        record = {
+            "category": PLATFORM,
+            "message_id": mid,
+            "server": meta.get("server"),
+            "server_id": meta.get("server_id"),
+            "channel": meta.get("channel"),
+            "channel_id": meta.get("channel_id") or post.get("channel_id"),
+            "parent": meta.get("parent"),
+            "is_thread": meta.get("is_thread"),
+            "author": author.get("username"),
+            "author_name": author.get("global_name"),
+            "author_id": author.get("id"),
+            "message": body,
+            "date": post.get("timestamp"),
+        }
+        folder = channel_dir(self.images_root, artist_slug, post)
+        folder.mkdir(parents=True, exist_ok=True)
+        when = message_date(post)
+        day = f"{when:%Y%m%d}" if when else "None"
+        path = folder / f"{day}_{mid}_post.json"
+        path.write_text(json.dumps(
+            {k: v for k, v in record.items() if v is not None}, indent=2,
+        ))
+        return PostRecordOutcome(
+            path=path, post_type=None, title=None, body_chars=len(body),
+        )

backend/app/services/discord_grouping.py

@@ -63,7 +63,7 @@ from collections import Counter
 from dataclasses import dataclass, field
 from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import Select, delete, func, select, update
+from sqlalchemy import Select, delete, func, select, union, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -126,6 +126,31 @@ def cosine_distance(a, b) -> float:
     return 1.0 - (dot / (na * nb))
 
 
+def _message_images(posts):
+    """(post_id, image_id) for every image a message carries — owned AND re-posted.
+
+    A message owns an image through `primary_post_id`, but only the FIRST
+    message imported with a given file does. The same file posted again is a
+    provenance link, and the backfill runs newest-first, so it is usually the
+    ORIGINAL message that ends up owning nothing. Reading ownership alone left
+    101 of Yellowroom's messages (2018–2020 mostly) ungroupable: every image
+    they carried also sat in another message.
+
+    `posts` is a list of ids or a select of them; filtering both branches by it
+    keeps the union to the messages in hand rather than the whole library.
+    Callers pass MESSAGE posts only — a drop's own provenance rows would read
+    as images it carries.
+    """
+    owned = select(
+        ImageRecord.primary_post_id.label("post_id"), ImageRecord.id.label("image_id"),
+    ).where(ImageRecord.primary_post_id.in_(posts))
+    reposted = select(
+        ImageProvenance.post_id.label("post_id"),
+        ImageProvenance.image_record_id.label("image_id"),
+    ).where(ImageProvenance.post_id.in_(posts))
+    return union(owned, reposted).subquery()
+
+
 def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
     """Ungrouped Discord message-posts, one representative image each, OLDEST
     FIRST — which is the order `build_groups` requires.
@@ -143,13 +168,15 @@ def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
     take the OLDEST candidates instead of the lowest-numbered ones.
     """
     sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
+    carried = _message_images(select(Post.id).where(Post.source_id == source_id))
     inner = (
         select(
             Post.id.label("post_id"),
             sort_key.label("occurred_at"),
             ImageRecord.siglip_embedding.label("embedding"),
         )
-        .join(ImageRecord, ImageRecord.primary_post_id == Post.id)
+        .join(carried, carried.c.post_id == Post.id)
+        .join(ImageRecord, ImageRecord.id == carried.c.image_id)
         .where(
             Post.source_id == source_id,
             # Never absorb a post FC wrote, and never re-absorb one already
@@ -295,9 +322,10 @@ async def _link_member_images(
     """
     if not member_ids:
         return 0
-    image_rows = (await session.execute(
-        select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids))
-    )).scalars().all()
+    carried = _message_images(member_ids)
+    image_rows = sorted(set((await session.execute(
+        select(carried.c.image_id)
+    )).scalars().all()))
     if not image_rows:
         return 0
     await session.execute(
@@ -421,9 +449,11 @@ async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None
     free to disagree; this way there is one.
     """
     sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
+    carried = _message_images(select(Post.id).where(Post.absorbed_by_post_id == post_id))
     return (await session.execute(
         select(ImageRecord.siglip_embedding)
-        .join(Post, ImageRecord.primary_post_id == Post.id)
+        .join(carried, carried.c.image_id == ImageRecord.id)
+        .join(Post, Post.id == carried.c.post_id)
         .where(
             Post.absorbed_by_post_id == post_id,
             ImageRecord.siglip_embedding.is_not(None),
@@ -725,12 +755,15 @@ async def _load_drops(session: AsyncSession, source: Source) -> list[_Drop]:
     names: dict[int, set[str]] = {pid: set() for pid in by_id}
     images: dict[int, list[int]] = {pid: [] for pid in by_id}
     if owner_of:
-        for iid, primary, path in (await session.execute(
-            select(ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path)
-            .where(ImageRecord.primary_post_id.in_(list(owner_of)))
+        carried = _message_images(list(owner_of))
+        for iid, message, path in (await session.execute(
+            select(ImageRecord.id, carried.c.post_id, ImageRecord.path)
+            .join(carried, carried.c.image_id == ImageRecord.id)
             .order_by(ImageRecord.id)
         )).all():
-            drop = owner_of[primary]
+            drop = owner_of[message]
+            if iid in images[drop]:
+                continue  # one file carried by two of the drop's messages
             images[drop].append(iid)
             if (name := leading_name(path)) is not None:
                 names[drop].add(name)
@@ -844,7 +877,12 @@ async def merge_trickles(
             if drop.first_at - earlier.last_at > gap:
                 continue
             shared = mine & gated(earlier.names)
-            if shared:
+            if set(drop.images) & set(earlier.images):
+                # The creator posted the very same file again — the strongest
+                # evidence there is, and one nearest-neighbour cannot see: it
+                # skips the image itself, which is the one they share.
+                targets[earlier.post.id] = (earlier, "same_image")
+            elif shared:
                 targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
             elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
                 targets[earlier.post.id] = (earlier, "nearest")

backend/app/services/discord_ingester.py

@@ -0,0 +1,133 @@
+"""Native Discord ingester — the Discord ADAPTER over `ingest_core.Ingester`.
+
+Thin counterpart to subscribestar_ingester (milestone 428). The walk's modes,
+both ledgers, cursor checkpointing and the post-first capture live in the core;
+this wires in the Discord client, downloader, ledger models and key.
+
+Two things differ from the cookie platforms:
+
+  - Discord authenticates with a user TOKEN, so `auth_token` is the credential
+    here rather than an argument accepted and ignored.
+  - The body canary is off. It fails a walk whose first 30+ captured posts all
+    came back without text, on the theory that a creator nearly always writes
+    something; a Discord drop is routinely files and nothing else, so on
+    Discord that is an ordinary backfill, not a broken parser.
+
+And two things only a Discord source has: `discord_authors` in its
+config_overrides limits it to the creator's own messages (a server's other
+members post pictures too), and it learns its own name — the server and
+channel it walks — into `source.display_name`.
+
+`campaign_id` is the source URL (a server, channel, thread or category link).
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from pathlib import Path
+
+from sqlalchemy import select, update
+
+from ..models import DiscordFailedMedia, DiscordSeenMedia, Source
+from .discord_client import DiscordAPIError, DiscordClient
+from .discord_downloader import DiscordDownloader
+from .ingest_core import Ingester
+from .native_ingest_common import ledger_key
+
+log = logging.getLogger(__name__)
+
+# `config_overrides` key: the people whose messages this source takes (user
+# ids, usernames or display names). Absent or empty takes everyone's.
+AUTHORS_KEY = "discord_authors"
+
+
+class DiscordIngester(Ingester):
+    """Walk a Discord source's channels, download unseen files, return a
+    `DownloadResult`. `client` / `downloader` are injectable for tests."""
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None,
+        session_factory: Callable[[], object],
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        request_sleep: float = 0.0,
+        auth_token: str | None = None,
+        client: DiscordClient | None = None,
+        downloader: DiscordDownloader | None = None,
+    ):
+        del cookies_path  # Discord authenticates by token (uniform signature)
+        self.images_root = Path(images_root)
+        super().__init__(
+            client=client if client is not None else DiscordClient(
+                auth_token, request_sleep=request_sleep,
+            ),
+            downloader=downloader if downloader is not None else DiscordDownloader(
+                self.images_root, validate=validate, rate_limit=rate_limit,
+            ),
+            session_factory=session_factory,
+            seen_model=DiscordSeenMedia,
+            failed_model=DiscordFailedMedia,
+            seen_constraint="uq_discord_seen_media_source_id",
+            failed_constraint="uq_discord_failed_media_source_id",
+            ledger_key=ledger_key,
+            platform="discord",
+            error_base=DiscordAPIError,
+            drift_label="Discord API",
+            body_canary=False,
+        )
+
+    def run(self, **kwargs):
+        """The core walk, bracketed by the two things only a Discord source
+        has: whose messages it takes, read before the walk, and the name of
+        what it walks, written after it (the walk is what loads that name)."""
+        source_id = kwargs["source_id"]
+        self.client.only_from(self._source_authors(source_id))
+        try:
+            return super().run(**kwargs)
+        finally:
+            self._record_display_name(source_id, kwargs["campaign_id"])
+
+    def _source_authors(self, source_id: int) -> list:
+        if self.session_factory is None:
+            return []
+        with self.session_factory() as session:
+            overrides = session.execute(
+                select(Source.config_overrides).where(Source.id == source_id)
+            ).scalar_one_or_none() or {}
+        authors = overrides.get(AUTHORS_KEY) or []
+        return authors if isinstance(authors, list) else []
+
+    def _record_display_name(self, source_id: int, url: str) -> None:
+        """Refreshed on every walk, never written once: a renamed channel
+        should read as its new name. A name the walk couldn't read leaves the
+        stored one alone, and a failure here never fails the walk."""
+        label = self.client.source_label(url)
+        if not label or self.session_factory is None:
+            return
+        try:
+            with self.session_factory() as session:
+                session.execute(
+                    update(Source)
+                    .where(Source.id == source_id)
+                    .where(Source.display_name.is_distinct_from(label))
+                    .values(display_name=label)
+                )
+                session.commit()
+        except Exception as exc:  # a name is decoration — never fail the walk
+            log.warning("Discord: couldn't record source %s's name: %s", source_id, exc)
+
+
+async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
+    """The uniform `(ok, message)` probe: is the token valid, and can its
+    account see the channel or server the source names?"""
+    if not auth_token:
+        return False, "No Discord token is saved — add one under Credentials."
+    client = DiscordClient(auth_token)
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, client.verify_auth, url)

backend/app/services/discord_poster_cleanup.py

@@ -0,0 +1,243 @@
+"""Remove a Discord source's posts by people outside its poster list (#4486).
+
+A creator's server is full of other members posting their own pictures. The
+poster list (`discord_authors`, #4481) stops a walk taking them; this removes
+the ones taken before the list existed. The operator sets the list first, then
+previews what would go, then applies.
+
+## What goes
+
+A post of this source whose record names its poster (`author_id` / `author`,
+written by `DiscordDownloader.write_post_record`) and names someone NOT on the
+list. Two kinds of post are never touched:
+
+* a post whose record names no poster — a message recorded before the native
+  ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
+* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
+  below as a consequence, not matched as a poster's post.
+
+An image goes when every REAL post it belongs to is going. A link to a
+synthetic drop does not keep an image: the drop only references its members'
+images. An image also on a kept post — the creator re-posting a piece someone
+else shared — stays.
+
+A drop that absorbed a removed message is deleted outright. Deleting a drop is
+its undo (discord_grouping's honesty rule): its remaining members return to
+the feed and the grouping sweep regroups them on its next run, so no half-
+rebuilt drop keeps text or thumbnails from someone it no longer contains.
+
+Preview and apply spread the same predicates (rule 93, snippet #3087).
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from sqlalchemy import and_, delete, exists, func, or_, select
+from sqlalchemy.orm import Session, aliased
+
+from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
+from .cleanup_service import delete_images
+from .discord_ingester import AUTHORS_KEY
+
+log = logging.getLogger(__name__)
+
+PLATFORM = "discord"
+# The record keys that name a message's poster. `author_name` (the display
+# name) is only on records written after 2026-09-28.
+_POSTER_KEYS = ("author_id", "author", "author_name")
+
+
+class PosterCleanupError(ValueError):
+    """The source can't be cleaned this way (not Discord, or no list)."""
+
+
+def source_authors(source: Source) -> list[str]:
+    """The source's poster list, lowercased; empty means everyone's wanted."""
+    authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
+    if not isinstance(authors, list):
+        return []
+    return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
+
+
+def _poster(key: str):
+    return func.lower(Post.raw_metadata[key].as_string())
+
+
+def _has_poster():
+    return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
+
+
+def _real_post_conditions(source_id: int) -> list:
+    return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
+
+
+def _named_poster_conditions(source_id: int) -> list:
+    """A real post of this source whose record says who posted it."""
+    return [*_real_post_conditions(source_id), _has_poster()]
+
+
+def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
+    """The posts that go: a named poster none of whose names is on the list."""
+    return [
+        *_named_poster_conditions(source_id),
+        *(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
+    ]
+
+
+def _doomed_post_ids(source_id: int, authors: list[str]):
+    # correlate(None): used inside queries that are themselves over `post` (the
+    # drops, the delete), where auto-correlation would bind this to the outer
+    # row instead of scanning the table.
+    return (
+        select(Post.id)
+        .where(*_other_poster_post_conditions(source_id, authors))
+        .correlate(None)
+    )
+
+
+def _image_conditions(source_id: int, authors: list[str]) -> list:
+    """Images that belong to a removed post and to no kept real post."""
+    doomed = _doomed_post_ids(source_id, authors)
+    keeper = aliased(Post)
+    on_doomed = or_(
+        ImageRecord.primary_post_id.in_(doomed),
+        exists().where(
+            ImageProvenance.image_record_id == ImageRecord.id,
+            ImageProvenance.post_id.in_(doomed),
+        ),
+    )
+    on_kept = or_(
+        and_(
+            ImageRecord.primary_post_id.isnot(None),
+            ImageRecord.primary_post_id.notin_(doomed),
+        ),
+        exists().where(
+            ImageProvenance.image_record_id == ImageRecord.id,
+            ImageProvenance.post_id == keeper.id,
+            keeper.synthesized_by.is_(None),
+            keeper.id.notin_(doomed),
+        ),
+    )
+    return [on_doomed, ~on_kept]
+
+
+def _drop_conditions(source_id: int, authors: list[str]) -> list:
+    """The synthetic drops that absorbed a removed message."""
+    member = aliased(Post)
+    return [
+        Post.source_id == source_id,
+        Post.synthesized_by.isnot(None),
+        exists().where(
+            member.absorbed_by_post_id == Post.id,
+            member.id.in_(_doomed_post_ids(source_id, authors)),
+        ),
+    ]
+
+
+def _attachment_conditions(source_id: int, authors: list[str]) -> list:
+    return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
+
+
+def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
+    source = session.get(Source, source_id)
+    if source is None:
+        raise LookupError(source_id)
+    if source.platform != PLATFORM:
+        raise PosterCleanupError("Only a Discord source has posters to filter by.")
+    authors = source_authors(source)
+    if not authors:
+        raise PosterCleanupError(
+            "Set this source's 'Only posts by' list first — with no list, "
+            "every poster is wanted and nothing would be removed."
+        )
+    return source, authors
+
+
+def _count(session: Session, stmt) -> int:
+    return session.execute(stmt).scalar_one()
+
+
+def preview(session: Session, *, source_id: int) -> dict:
+    """What `apply` would remove, per poster, without touching anything."""
+    _, authors = _load(session, source_id)
+    who = func.coalesce(
+        Post.raw_metadata["author_name"].as_string(),
+        Post.raw_metadata["author"].as_string(),
+        Post.raw_metadata["author_id"].as_string(),
+    )
+    rows = session.execute(
+        select(who, func.count(Post.id))
+        .where(*_other_poster_post_conditions(source_id, authors))
+        .group_by(who)
+        .order_by(func.count(Post.id).desc())
+    ).all()
+    posters = [{"poster": name, "posts": n} for name, n in rows]
+    unknown = _count(session, select(func.count(Post.id)).where(
+        *_real_post_conditions(source_id), ~_has_poster(),
+    ))
+    return {
+        "authors": authors,
+        "posters": posters,
+        "posts": sum(p["posts"] for p in posters),
+        "images": _count(session, select(func.count(ImageRecord.id)).where(
+            *_image_conditions(source_id, authors))),
+        "attachments": _count(session, select(func.count(PostAttachment.id)).where(
+            *_attachment_conditions(source_id, authors))),
+        "drops": _count(session, select(func.count(Post.id)).where(
+            *_drop_conditions(source_id, authors))),
+        "unknown_posts": unknown,
+    }
+
+
+def confirm_token(projection: dict) -> str:
+    """What the operator's apply must echo: the preview it was shown, so a
+    list edited between preview and apply can't delete an unseen set."""
+    return f"remove-{projection['posts']}-posts-{projection['images']}-images"
+
+
+def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
+    """Remove them. Counts are taken before the deletes they describe — once
+    the rows are gone there is nothing left to count (snippet #3087 note 2)."""
+    projection = preview(session, source_id=source_id)
+    _, authors = _load(session, source_id)
+
+    image_ids = session.execute(
+        select(ImageRecord.id).where(*_image_conditions(source_id, authors))
+    ).scalars().all()
+    # Drops first-class: found BEFORE their members go, since the link that
+    # finds them is the member's absorbed_by_post_id.
+    drop_ids = session.execute(
+        select(Post.id).where(*_drop_conditions(source_id, authors))
+    ).scalars().all()
+
+    deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
+    # Attachments before posts: post_attachment.post_id is SET NULL, and a
+    # NULL-post attachment collides on its partial unique index (see
+    # cleanup_service.delete_artist_cascade).
+    attachments = session.execute(
+        delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
+    ).rowcount or 0
+    posts = session.execute(
+        Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
+    ).rowcount or 0
+    drops = 0
+    if drop_ids:
+        drops = session.execute(
+            Post.__table__.delete().where(Post.id.in_(drop_ids))
+        ).rowcount or 0
+    session.commit()
+    log.info(
+        "discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
+        "%d attachment(s), %d drop(s) removed",
+        source_id, authors, posts, deleted["images_deleted"], attachments, drops,
+    )
+    return {
+        **projection,
+        "posts_deleted": posts,
+        "images_deleted": deleted["images_deleted"],
+        "files_deleted": deleted["files_deleted"],
+        "attachments_deleted": attachments,
+        "drops_deleted": drops,
+    }

backend/app/services/download_backends.py

@@ -23,7 +23,8 @@ from __future__ import annotations
 import asyncio
 from pathlib import Path
 
-from .gallery_dl import DownloadResult, ErrorType
+from .discord_ingester import DiscordIngester
+from .download_result import DownloadResult, ErrorType
 from .ingest_core import DEFAULT_REVISIT_DAYS
 from .patreon_ingester import PatreonIngester
 from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
@@ -31,9 +32,13 @@ from .platforms import known_platform_keys
 from .subscribestar_ingester import SubscribeStarIngester
 
 # Platforms whose download + verify go through the native ingester rather than
-# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
-# they migrate too.
-NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
+# gallery-dl. gallery-dl still serves the rest (hentaifoundry) until it
+# migrates too. Discord joined in milestone 428.
+NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
+
+# Native platforms whose feed id IS the source URL, so there is nothing to
+# resolve: SubscribeStar's creator page, Discord's server/channel link.
+_URL_IS_FEED = frozenset({"subscribestar", "discord"})
 
 
 def _unsupported_platform_message(platform: str) -> str | None:
@@ -67,6 +72,7 @@ def _native_ingester_cls(platform: str):
     return {
         "patreon": PatreonIngester,
         "subscribestar": SubscribeStarIngester,
+        "discord": DiscordIngester,
     }[platform]
 
 
@@ -126,10 +132,10 @@ async def _resolve_native_campaign_id(
     platform: str, url: str, cookies_path: str | None, overrides: dict,
 ) -> tuple[str | None, str | None]:
     """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
-    feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
-    campaign id from the vanity URL (resolved non-None when a lookup actually ran,
-    so phase 3 caches it)."""
-    if platform == "subscribestar":
+    and Discord's feed id IS the source URL (no lookup → resolved None). Patreon
+    resolves the campaign id from the vanity URL (resolved non-None when a lookup
+    actually ran, so phase 3 caches it)."""
+    if platform in _URL_IS_FEED:
         return url, None
     return await resolve_campaign_id_for_source(url, cookies_path, overrides)
 
@@ -252,6 +258,10 @@ async def verify_source_credential(
             from .subscribestar_ingester import verify_subscribestar_credential
 
             return await verify_subscribestar_credential(url, cookies_path, config_overrides)
+        if platform == "discord":
+            from .discord_ingester import verify_discord_credential
+
+            return await verify_discord_credential(url, auth_token)
         from .patreon_ingester import verify_patreon_credential
 
         return await verify_patreon_credential(url, cookies_path, config_overrides)

backend/app/services/download_recovery.py

@@ -0,0 +1,57 @@
+"""Closing the download runs a worker restart orphaned (#4433).
+
+Its own module, importing nothing but the model, because its caller is the
+worker boot hook in `celery_signals` — which every download task imports via
+`celery_app`. Living in `tasks.maintenance` put the whole maintenance import
+graph, the membership roster included, on the fetch path, which
+`test_gated_reason` forbids.
+"""
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import literal, update
+from sqlalchemy.dialects.postgresql import JSONB
+
+from ..models import DownloadEvent
+
+DOWNLOAD_INTERRUPTED_MESSAGE = (
+    "interrupted by a worker restart — the next check picks it up where it left off"
+)
+
+
+def interrupt_orphaned_download_events(session, *, booted_at: datetime) -> int:
+    """Close the download events a restart orphaned, without blaming the source.
+
+    Called when the download lane comes up (#4433). Anything still
+    pending/running from before this boot belongs to the previous process:
+    a walk that outlived the 90s stop grace was SIGKILLed, and a queued or
+    serialize-deferred task is held unacked until Redis redelivers it about an
+    hour later. Left alone, the 30-min stall sweep would error each one and
+    bump `consecutive_failures`, backing the source off as if the platform had
+    failed.
+
+    Instead they end as `skipped` (terminal, not a failure) and the source is
+    not touched: `last_checked_at` keeps its old value, so the next tick finds
+    it due and the walk resumes from its checkpoint. A redelivered message
+    that arrives later finds no pending event and opens a fresh one.
+
+    An event promoted to running after the boot has `started_at` reset to its
+    real start (download_service), so it is never caught here. Does NOT commit.
+    """
+    now = datetime.now(UTC)
+    result = session.execute(
+        update(DownloadEvent)
+        .where(DownloadEvent.status.in_(["pending", "running"]))
+        .where(DownloadEvent.started_at < booted_at)
+        .values(
+            status="skipped",
+            finished_at=now,
+            error=DOWNLOAD_INTERRUPTED_MESSAGE,
+            metadata_=DownloadEvent.metadata_.op("||")(
+                literal({"error_type": "interrupted"}, JSONB)
+            ),
+        )
+        .returning(DownloadEvent.id)
+    )
+    return len(result.all())

backend/app/services/download_result.py

@@ -0,0 +1,222 @@
+"""What a download walk reports, whichever backend ran it.
+
+The native ingesters (Patreon, SubscribeStar, Discord) and the gallery-dl
+subprocess (HentaiFoundry) both return a `DownloadResult`, classify failures
+with the same `ErrorType`s and build `run_stats` through `make_run_stats`, so
+phase 3, the source's status chip and the Logs UI read one shape from either.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from enum import StrEnum
+
+
+class ErrorType(StrEnum):
+    AUTH_ERROR = "auth_error"
+    RATE_LIMITED = "rate_limited"
+    NOT_FOUND = "not_found"
+    ACCESS_DENIED = "access_denied"
+    NETWORK_ERROR = "network_error"
+    NO_NEW_CONTENT = "no_new_content"
+    TIER_LIMITED = "tier_limited"
+    TIMEOUT = "timeout"
+    HTTP_ERROR = "http_error"
+    UNSUPPORTED_URL = "unsupported_url"
+    VALIDATION_FAILED = "validation_failed"
+    # Native Patreon ingester only (plan #697): a response parsed as JSON but
+    # didn't match the JSON:API shape the ingester depends on (missing `data`,
+    # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
+    # is updating the ingester's field-set/parser, not rotating credentials. The
+    # contract test guards against silently shipping this.
+    API_DRIFT = "api_drift"
+    # Run made real progress (downloaded ≥1 file) but did not finish in the
+    # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
+    # mapping classifies this as "ok" because the next tick continues.
+    PARTIAL = "partial"
+    UNKNOWN_ERROR = "unknown_error"
+
+
+@dataclass
+class DownloadResult:
+    success: bool
+    url: str
+    artist_slug: str
+    platform: str
+    files_downloaded: int = 0
+    files_quarantined: int = 0
+    quarantined_paths: list[str] = field(default_factory=list)
+    written_paths: list[str] = field(default_factory=list)
+    # Native ingester only: post-only sidecar paths for posts that have NO
+    # downloadable media (pure-text posts), so the importer can still upsert the
+    # Post + its body. Empty on the gallery-dl path. Phase 3 imports these via
+    # Importer.upsert_post_record (keyed on external_post_id → updates, never
+    # doubles, the same Post a media import would create).
+    post_record_paths: list[str] = field(default_factory=list)
+    # Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
+    # pairs for already-present media whose ImageRecord.source_filehash should be
+    # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
+    # on the gallery-dl path and outside recapture.
+    relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
+    # Native ingester only: one entry per `written_paths` entry, the backfill
+    # cursor of the page that file came from. Phase 3 resumes the next chunk
+    # there when it leaves files unimported at its deadline. Empty on gallery-dl.
+    written_cursors: list[str | None] = field(default_factory=list)
+    stdout: str = ""
+    stderr: str = ""
+    return_code: int = 0
+    error_type: ErrorType | None = None
+    error_message: str | None = None
+    duration_seconds: float = 0.0
+    started_at: str | None = None
+    completed_at: str | None = None
+    # Plan #704 — structured fields the NATIVE ingester populates directly (None
+    # on the gallery-dl path, which keeps the regex-over-stdout route). When set,
+    # phase 3 reads these instead of scraping the text it would otherwise have to
+    # reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the
+    # backfill checkpoint the ingester knows exactly (no parse_last_cursor).
+    run_stats: dict | None = None
+    cursor: str | None = None
+    posts_processed: int = 0
+    # Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so
+    # the platform cooldown matches the hint instead of a flat default. None when
+    # unknown (no header, or not a rate-limit failure).
+    retry_after_seconds: float | None = None
+    # Native ingester only: marks this walk's fetched media seen in its ledger.
+    # Phase 3 calls it AFTER the import loop, never before — a file marked seen
+    # but not yet imported is invisible to every later walk, so a run killed in
+    # between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13
+    # images to a stranded run, 2026-09-24). Unmarked, the next walk finds the
+    # file on disk with no ImageRecord and imports it. Takes `skip_paths`: the
+    # written paths phase 3 did not import, which stay unmarked for the same
+    # reason. None on gallery-dl.
+    mark_seen_after_import: Callable[..., None] | None = None
+
+
+def extract_errors_warnings(stderr: str) -> str:
+    """Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
+
+    A generic download-log helper (module-level so the native-ingester result
+    path can shape its logs without reaching through a GalleryDLService instance).
+    """
+    if not stderr:
+        return ""
+    kept = [
+        line for line in stderr.splitlines()
+        if "][error]" in line.lower() or "][warning]" in line.lower()
+    ]
+    return "\n".join(kept)
+
+
+def truncate_log(text: str, max_bytes: int = 500_000) -> str:
+    """Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
+    if not text:
+        return text
+    encoded = text.encode("utf-8")
+    if len(encoded) <= max_bytes:
+        return text
+    half = max_bytes // 2
+    head = encoded[:half].decode("utf-8", errors="ignore")
+    tail = encoded[-half:].decode("utf-8", errors="ignore")
+    head_lines = head.count("\n")
+    tail_lines = tail.count("\n")
+    total_lines = text.count("\n")
+    elided = max(0, total_lines - head_lines - tail_lines)
+    marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
+    return head + marker + tail
+
+
+def make_run_stats(
+    *,
+    exit_code: int = 0,
+    downloaded_count: int = 0,
+    skipped_count: int = 0,
+    per_item_failures: int = 0,
+    warning_count: int = 0,
+    tier_gated_count: int = 0,
+    quarantined_count: int = 0,
+    dead_lettered_count: int = 0,
+) -> dict:
+    """The canonical `run_stats` dict shape, in ONE place.
+
+    Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
+    native ingester's per-outcome tally (ingest_core) — build through this so the
+    key set can't drift between backends. Phase 3 + the Logs UI read these keys.
+    """
+    return {
+        "exit_code": exit_code,
+        "downloaded_count": downloaded_count,
+        "skipped_count": skipped_count,
+        "per_item_failures": per_item_failures,
+        "warning_count": warning_count,
+        "tier_gated_count": tier_gated_count,
+        "quarantined_count": quarantined_count,
+        "dead_lettered_count": dead_lettered_count,
+    }
+
+
+# --- tier-gated classification, shared by BOTH backends ---------------------
+#
+# These three live together because the native ingester and the gallery-dl
+# subprocess must reach the same verdict from the same number. They did not:
+# gallery-dl classified TIER_LIMITED while ingest_core counted gated posts and
+# threw the count away, so the platforms FC owns reported a paywalled creator as
+# a silent one (#874 follow-up). One predicate, spread into both, rather than
+# the condition re-derived per backend.
+
+
+def classify_tier_gated(tier_gated_count: int) -> ErrorType | None:
+    """TIER_LIMITED when a walk saw tier-gated posts and nothing else failed.
+
+    Deliberately NOT conditioned on `downloaded == 0`. A creator whose top-tier
+    posts we cannot see is tier-limited even in a week we did get their cheaper
+    ones — the fact the operator needs ("there is content here you are not
+    paying for") is true either way. gallery-dl has classified it this way since
+    the paywall-as-"needs attention" complaint (see `_categorize_error`), and the
+    native path now matches rather than inventing a stricter rule.
+
+    Callers must apply this only AFTER the real error categories (auth, rate
+    limit, drift, …) have had their turn; tier-gating is the weakest signal and
+    must never mask a genuine failure.
+    """
+    return ErrorType.TIER_LIMITED if tier_gated_count else None
+
+
+def tier_gated_message(count: int) -> str:
+    """The one wording for the tier-gated verdict, so the two backends can't
+    describe the same state differently in the Logs UI."""
+    return (
+        f"Subscription tier does not grant access to "
+        f"{count} post{'s' if count != 1 else ''}"
+    )
+
+
+# `Source.error_type` doubles as the failure-class chip, and a status of "ok"
+# CLEARS it (alembic 0032). TIER_LIMITED breaks that assumption: it rides an
+# otherwise-successful run, so without an exemption the chip is wiped the moment
+# it is set and `FailingSourcesCard`'s `tier_limited` palette entry can never
+# render. Informational classes are the exemption — they describe the source,
+# not a failure of the run.
+INFORMATIONAL_ERROR_TYPES = frozenset({ErrorType.TIER_LIMITED.value})
+
+
+def is_informational(error_type) -> bool:
+    """True for a class that reports a state rather than a failure. Accepts an
+    ErrorType or the plain string persisted on Source.error_type."""
+    return error_type is not None and str(error_type) in INFORMATIONAL_ERROR_TYPES
+
+
+def walk_completed(result: DownloadResult) -> bool:
+    """Did this walk reach the bottom cleanly?
+
+    The backfill lifecycle's completion test. An informational error_type still
+    counts as complete: a fully-paywalled creator's backfill DID finish, and
+    treating it as unfinished re-walks the same wall until the stall counter
+    trips — the creator we can see least becoming the one we fetch most.
+    """
+    return (
+        result.success
+        and result.return_code == 0
+        and (result.error_type is None or is_informational(result.error_type))
+    )

backend/app/services/download_service.py

@@ -3,8 +3,9 @@
 Three-phase pipeline:
   Phase 1 — DB setup (brief session): load source, mark event running,
             fetch credentials + settings.
-  Phase 2 — Execute (no DB connection): call GalleryDLService; on
-            Patreon campaign-ID failure, resolve vanity and retry once.
+  Phase 2 — Execute (no DB connection): run the source's backend (a native
+            ingester, or gallery-dl for HentaiFoundry); on a Patreon
+            campaign-ID failure, resolve the vanity and retry once.
   Phase 3 — Import + persist (fresh session): attach_in_place each
             written file; clean up duplicates; persist rich metadata.
 """
@@ -13,6 +14,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import time
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -25,27 +27,39 @@ from sqlalchemy.orm import joinedload
 from ..models import Artist, DownloadEvent, Source
 from .credential_service import CredentialService
 from .download_backends import run_download, uses_native_ingester
-from .gallery_dl import (
-    BACKFILL_CHUNK_SECONDS,
-    BACKFILL_SKIP_VALUE,
-    TICK_SKIP_VALUE,
+from .download_result import (
     DownloadResult,
     ErrorType,
-    GalleryDLService,
-    SourceConfig,
     extract_errors_warnings,
     is_informational,
     truncate_log,
     walk_completed,
 )
+from .gallery_dl import (
+    BACKFILL_CHUNK_SECONDS,
+    BACKFILL_SKIP_VALUE,
+    TICK_SKIP_VALUE,
+    GalleryDLService,
+    SourceConfig,
+)
 from .importer import Importer
-from .ingest_core import DEFAULT_REVISIT_DAYS
+from .ingest_core import DEFAULT_REVISIT_DAYS, PHASE3_DEADLINE_SECONDS
 from .platforms import auth_type_for
 from .scheduler_service import set_platform_cooldown
 
 log = logging.getLogger(__name__)
 
 
+def credential_proven(ctx: dict, dl_result) -> bool:
+    """Did this walk prove the platform's stored credential still works?
+
+    A clean walk made with a credential is the evidence the Verify button looks
+    for. A walk with any error type proves nothing, tier-limited included:
+    content hidden behind a tier is also what an expired session looks like."""
+    used = bool(ctx.get("cookies_path") or ctx.get("auth_token"))
+    return used and bool(dl_result.success) and dl_result.error_type is None
+
+
 class DownloadService:
     """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
 
@@ -82,6 +96,7 @@ class DownloadService:
 
     async def download_source(self, source_id: int) -> int:
         """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
+        started = time.monotonic()
         setup = await self._phase1_setup(source_id)
         if setup["status"] in ("skipped", "in_flight"):
             return setup["event_id"]
@@ -174,6 +189,7 @@ class DownloadService:
 
         return await self._phase3_persist(
             ctx["event_id"], ctx, dl_result, resolved_campaign_id,
+            started=started,
         )
 
     async def _run_download(
@@ -268,6 +284,8 @@ class DownloadService:
         ctx: dict,
         dl_result,
         resolved_campaign_id: str | None,
+        *,
+        started: float | None = None,
     ) -> int:
         # Cache Patreon campaign id on the source.
         if resolved_campaign_id:
@@ -303,7 +321,23 @@ class DownloadService:
 
         loop = asyncio.get_running_loop()
 
-        for path_str in dl_result.written_paths or []:
+        # Phase 3 shares the task's soft limit with the walk, and an import's
+        # cost is only estimated there. Native platforms stop starting imports
+        # at the deadline: what is left stays on disk with no record, which the
+        # next chunk's walk imports. gallery-dl platforms have no such pickup,
+        # so they import everything.
+        deadline = (
+            started + PHASE3_DEADLINE_SECONDS
+            if started is not None and uses_native_ingester(ctx["platform"])
+            else None
+        )
+        written_paths = list(dl_result.written_paths or [])
+        deferred_from: int | None = None
+
+        for idx, path_str in enumerate(written_paths):
+            if deadline is not None and time.monotonic() >= deadline:
+                deferred_from = idx
+                break
             if path_str in (dl_result.quarantined_paths or []):
                 continue
             path = Path(path_str)
@@ -348,11 +382,8 @@ class DownloadService:
                     if do_embed:
                         embed_image.delay(img_id)
             elif result.status == "attached":
-                # Non-media or extracted archive captured as PostAttachment
-                # (FC-2d-iii). The canonical copy lives in the attachments
-                # store; the original download path is now redundant —
-                # mirror duplicate_hash cleanup so we don't keep two copies.
-                # Operator-flagged 2026-06-02 (Lustria OST zip).
+                # Non-media, or an archive with no new images, captured as a
+                # PostAttachment; the attachments store holds the copy.
                 import_summary["attached"] += 1
                 # An archive captured WITHOUT extracting any image carries a
                 # reason on result.error — record it so the event explains the
@@ -369,35 +400,13 @@ class DownloadService:
                     bytes_downloaded += path.stat().st_size  # noqa: ASYNC240
                 except OSError:
                     pass
-                try:
-                    path.unlink(missing_ok=True)  # noqa: ASYNC240
-                except OSError:
-                    pass
-            elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
-                "duplicate_hash", "duplicate_phash",
-            ):
-                import_summary["skipped"] += 1
-                try:
-                    path.unlink(missing_ok=True)  # noqa: ASYNC240
-                except OSError:
-                    pass
             elif result.status == "skipped":
-                # Soft skip (too_small, too_transparent, invalid_image) —
-                # the file just didn't qualify, not a download/ingest
-                # failure. Don't flag the run as error; the file stays
-                # on disk for operator inspection.
+                # A duplicate, or a file the import filters rejected. Neither
+                # is a download failure, so the run is not flagged.
                 import_summary["skipped"] += 1
             elif result.status == "failed":
-                # Hard failure (today only: archive probe crash/timeout).
-                # The original archive sits in /images/ as an orphan; the
-                # filesystem scanner would re-import and re-crash on the
-                # same file, so delete the source file and surface the
-                # error in import_summary. Audit 2026-06-02.
+                # Hard failure: an archive probe crashed or timed out.
                 import_summary["errors"] += 1
-                try:
-                    path.unlink(missing_ok=True)  # noqa: ASYNC240
-                except OSError:
-                    pass
             elif result.status == "refreshed":
                 # Currently unreachable from attach_in_place (the download
                 # path never runs in deep=True mode), but the importer's
@@ -406,6 +415,10 @@ class DownloadService:
                 import_summary["attached"] += 1
             else:
                 import_summary["errors"] += 1
+            # Remove the download when the library no longer needs it at this
+            # path: an attachment or archive now in the store, a duplicate, a
+            # rejected file. One policy for every download path.
+            self.importer.discard_after_attach(path, result)
 
         # Post-only records: media-less posts (pure-text) the native ingester
         # captured so the artist archive is complete. Upsert each (keyed on
@@ -423,6 +436,21 @@ class DownloadService:
 
             await loop.run_in_executor(None, _upsert)
 
+        # Only now is it safe to call this walk's media seen: every file above
+        # has been through the importer. Had the run died before here they stay
+        # unmarked, and the next walk imports them from disk (ingest_core).
+        mark_seen = getattr(dl_result, "mark_seen_after_import", None)
+        deferred = written_paths[deferred_from:] if deferred_from is not None else []
+        if mark_seen is not None:
+            if deferred:
+                await loop.run_in_executor(
+                    None, lambda: mark_seen(skip_paths=frozenset(deferred)),
+                )
+            else:
+                await loop.run_in_executor(None, mark_seen)
+        if deferred:
+            self._defer_unimported(dl_result, deferred_from, len(deferred))
+
         # #830 recapture: backfill source_filehash on EXISTING on-disk images so
         # their post-body inline <img src=CDN> remaps to the local copy. A
         # SEPARATE non-deleting channel (NOT the import list — that would unlink
@@ -521,8 +549,39 @@ class DownloadService:
         )
         await self._apply_backfill_lifecycle(ctx, dl_result)
         await self.async_session.commit()
+        if credential_proven(ctx, dl_result):
+            # The credentials page then shows when the account last worked,
+            # not only when someone last pressed Verify.
+            await self.cred_service.mark_verified(ctx["platform"])
         return event_id
 
+    @staticmethod
+    def _defer_unimported(dl_result, first: int, count: int) -> None:
+        """Turn this chunk into a chunk boundary at the first file phase 3 left.
+
+        The next chunk resumes on the page that file came from, so its walk
+        meets every left file again, on disk with no record, and imports it. A
+        walk that reached its end has not finished while files wait, so a clean
+        result becomes PARTIAL; a real failure keeps its own error.
+        """
+        cursors = getattr(dl_result, "written_cursors", None) or []
+        if first < len(cursors):
+            dl_result.cursor = cursors[first]
+        dl_result.phase3_imported_any = first > 0
+        msg = (
+            f"Imports stopped at the chunk deadline: {count} file(s) left "
+            "for the next chunk"
+        )
+        dl_result.stdout = f"{dl_result.stdout}\n{msg}" if dl_result.stdout else msg
+        log.info("%s", msg)
+        if dl_result.error_type in (None, ErrorType.PARTIAL, ErrorType.TIMEOUT) or (
+            is_informational(dl_result.error_type)
+        ):
+            dl_result.success = False
+            dl_result.return_code = -1
+            dl_result.error_type = ErrorType.PARTIAL
+            dl_result.error_message = msg
+
     async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None:
         """Backfill state machine (plan #693, building on the cursor of #689).
 
@@ -590,9 +649,12 @@ class DownloadService:
         new_cursor = (
             dl_result.cursor if uses_native_ingester(ctx["platform"]) else None
         )
+        # Imports phase 3 finished before its deadline are progress even when
+        # the cursor went back to the page holding the files it left.
         advanced = bool(
             (new_cursor and new_cursor != old_cursor)
             or dl_result.files_downloaded > 0
+            or getattr(dl_result, "phase3_imported_any", False)
         )
         if advanced:
             if new_cursor:

backend/app/services/extension_service.py

@@ -21,6 +21,15 @@ from .source_service import BACKFILL_MAX_CHUNKS
 
 log = logging.getLogger(__name__)
 
+# The probe runs while the chip is drawing; names that take longer are skipped.
+_NAME_LOOKUP_SECONDS = 6.0
+# The poster picker waits on two pages of a channel's messages (#4491).
+_POSTER_LOOKUP_SECONDS = 20.0
+
+# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this
+# keeps the name each id was picked under, for showing the list to a person.
+AUTHOR_LABELS_KEY = "discord_author_labels"
+
 
 class UnknownPlatformError(Exception):
     """URL didn't match any platform pattern."""
@@ -30,6 +39,19 @@ class InvalidUrlError(Exception):
     """URL was empty or missing a scheme."""
 
 
+class UnknownArtistError(Exception):
+    """quick-add named an `artist_id` that does not exist."""
+
+
+class PosterLookupError(Exception):
+    """The Discord poster picker couldn't read the channel (no token, no
+    access, not a channel page, or Discord was slow)."""
+
+
+class UnknownSourceError(Exception):
+    """No source follows this Discord channel or its server."""
+
+
 # Mirrored byte-for-byte from extension/lib/platforms.js
 # PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
 # reviewers catch drift.
@@ -55,8 +77,39 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
         r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
         re.IGNORECASE,
     )),
+    # A Discord URL names a server or a channel, never a creator, so the slug is
+    # `<server>` or `<server>/<channel>` and the artist is chosen, not derived.
+    # A trailing message id (a jump link) still names its channel. DMs (`@me`)
+    # are not sources; thread links (`/threads/`) are left to the manual form.
+    ("discord", re.compile(
+        r"^https?://(?:www\.|ptb\.|canary\.)?discord\.com/channels/"
+        r"(?P<slug>\d+(?:/\d+)?)(?:/\d+)?/?(?:[?#].*)?$",
+        re.IGNORECASE,
+    )),
 ]
 
+DISCORD = "discord"
+
+
+def canonical_source_url(platform: str, url: str, slug: str) -> str:
+    """The URL a new source is stored under. Discord's is rebuilt from the ids
+    — the form the manual Add form and the ingester use — so a jump link, a
+    ptb/canary host or a trailing slash never makes a second source for the
+    same channel. Every other platform keeps the URL as given."""
+    if platform == DISCORD:
+        return f"https://discord.com/channels/{slug}"
+    return url
+
+
+def _discord_ids(url: str) -> tuple[str | None, str | None] | None:
+    """`(server_id, channel_id)` of a stored Discord source URL, None if it
+    does not parse (a DM or thread link, or an old malformed row)."""
+    from .discord_client import DiscordAPIError, parse_source_url
+    try:
+        return parse_source_url(url)
+    except DiscordAPIError:
+        return None
+
 
 class ExtensionService:
     def __init__(self, session: AsyncSession, crypto=None) -> None:
@@ -65,29 +118,179 @@ class ExtensionService:
         # add-time. None → skip resolution, fall back to the handle.
         self._crypto = crypto
 
-    async def quick_add_source(self, url: str) -> dict:
+    async def quick_add_source(
+        self,
+        url: str,
+        *,
+        artist_id: int | None = None,
+        artist_name: str | None = None,
+        use_platform_name: bool = False,
+        discord_authors: list[dict] | None = None,
+    ) -> dict:
+        """Add `url` as a source. `artist_id` connects it to an existing
+        artist, `artist_name` to that artist (created if new); with neither,
+        the artist is resolved from the platform as before.
+
+        `use_platform_name` applies the operator's convention that the Patreon
+        name is canon: a Patreon source added to an existing artist renames
+        that artist to the creator's Patreon display name. Name only — the
+        slug, and every path keyed off it, never moves (#130). Ignored on every
+        other platform, and when the name can't be read.
+
+        `discord_authors` ([{id, name}], #4491) sets a Discord source's poster
+        list in the same step, on a new source or one already there."""
         platform, raw_slug = self._derive(url)
+        url = canonical_source_url(platform, url, raw_slug)
+        renamed_from = None
         # Identity by SOURCE handle (#130): an existing (platform, url) source
         # keeps its artist on re-add — even if that artist was since renamed (its
-        # frozen slug no longer matches the current name). Only a genuinely new
-        # source resolves/creates an artist.
-        existing = (await self.session.execute(
-            select(Source).where(Source.platform == platform, Source.url == url)
-        )).scalar_one_or_none()
+        # frozen slug no longer matches the current name), and even when the
+        # add named a different artist. Only a genuinely new source
+        # resolves/creates an artist.
+        existing = await self._existing_source(platform, url)
         if existing is not None:
             artist = (await self.session.execute(
                 select(Artist).where(Artist.id == existing.artist_id)
             )).scalar_one()
+            if platform == DISCORD and discord_authors is not None:
+                await self._apply_posters(existing, discord_authors)
             return self._shape(existing, artist, created_source=False, created_artist=False)
 
-        # New source → name the artist properly by resolving the real display
-        # name from the platform (falls back to the URL handle).
-        name = await self._resolve_artist_name(platform, raw_slug, url)
-        artist, created_artist = await self._find_or_create_artist(name)
+        if artist_id is not None:
+            artist = (await self.session.execute(
+                select(Artist).where(Artist.id == artist_id)
+            )).scalar_one_or_none()
+            if artist is None:
+                raise UnknownArtistError(f"no artist with id {artist_id}")
+            created_artist = False
+            if use_platform_name and platform == "patreon":
+                renamed_from = await self._adopt_patreon_name(artist, raw_slug, url)
+        else:
+            name = (artist_name or "").strip()
+            if not name:
+                # Name the artist properly by resolving the real display name
+                # from the platform (falls back to the URL handle).
+                name = await self._resolve_artist_name(platform, raw_slug, url)
+            artist, created_artist = await self._find_or_create_artist(name)
         source, created_source = await self._find_or_create_source(
             artist_id=artist.id, platform=platform, url=url,
         )
-        return self._shape(source, artist, created_source, created_artist)
+        if platform == DISCORD and discord_authors is not None:
+            await self._apply_posters(source, discord_authors)
+        shaped = self._shape(source, artist, created_source, created_artist)
+        if renamed_from is not None:
+            shaped["renamed_from"] = renamed_from
+        return shaped
+
+    # -- Discord poster picker (#4491) ------------------------------------------
+
+    def _discord_page(self, url: str) -> tuple[str, str]:
+        platform, raw_slug = self._derive(url)
+        if platform != DISCORD:
+            raise InvalidUrlError("not a Discord page")
+        server_id, _, channel_id = raw_slug.partition("/")
+        if not channel_id:
+            raise PosterLookupError("Open a channel to see who posts in it.")
+        return server_id, channel_id
+
+    async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None:
+        """The source this channel's posts land on: the channel's own, else a
+        whole-server source that walks it."""
+        base = f"https://discord.com/channels/{server_id}"
+        return (
+            await self._existing_source(DISCORD, f"{base}/{channel_id}")
+            or await self._existing_source(DISCORD, base)
+        )
+
+    async def discord_posters(self, url: str) -> dict:
+        """Who posts in the channel `url` shows, newest `_POSTER_SCAN`
+        messages deep, with the likely creator marked — and, when a source
+        already follows it, the poster list that source has now."""
+        server_id, channel_id = self._discord_page(url)
+        import asyncio
+
+        from .credential_service import CredentialService
+        from .discord_client import DiscordAPIError, DiscordClient
+        token = None
+        if self._crypto is not None:
+            token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
+        if not token:
+            raise PosterLookupError("No Discord token is saved in FabledCurator.")
+        client = DiscordClient(token, max_retries=0)
+        loop = asyncio.get_running_loop()
+        try:
+            found = await asyncio.wait_for(
+                loop.run_in_executor(None, client.recent_posters, server_id, channel_id),
+                timeout=_POSTER_LOOKUP_SECONDS,
+            )
+        except TimeoutError as exc:
+            raise PosterLookupError("Discord took too long to answer; try again.") from exc
+        except DiscordAPIError as exc:
+            raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc
+        source = await self._source_for_page(server_id, channel_id)
+        co = (source.config_overrides or {}) if source is not None else {}
+        from .discord_ingester import AUTHORS_KEY
+        return {
+            **found,
+            "source_id": source.id if source is not None else None,
+            "selected": list(co.get(AUTHORS_KEY) or []),
+            # So someone on the list who hasn't posted lately still shows by
+            # name, and can be unticked.
+            "selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}),
+        }
+
+    async def set_discord_posters(self, url: str, authors: list[dict]) -> dict:
+        """Store the picked posters on the source that follows `url`'s channel."""
+        server_id, channel_id = self._discord_page(url)
+        source = await self._source_for_page(server_id, channel_id)
+        if source is None:
+            raise UnknownSourceError("No source follows this channel yet.")
+        return await self._apply_posters(source, authors)
+
+    async def _apply_posters(self, source: Source, authors: list[dict]) -> dict:
+        """Ids are what the list keys on — a username can change on a whim,
+        an id can't — and the name each was picked under rides beside it for
+        showing the list to a person. An empty pick clears the list, which
+        takes everyone's posts again."""
+        from .discord_ingester import AUTHORS_KEY
+        picked = [a for a in authors if str(a.get("id") or "").strip()]
+        co = dict(source.config_overrides or {})
+        if picked:
+            co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked]
+            co[AUTHOR_LABELS_KEY] = {
+                str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked
+            }
+        else:
+            co.pop(AUTHORS_KEY, None)
+            co.pop(AUTHOR_LABELS_KEY, None)
+        source.config_overrides = co or None
+        await self.session.commit()
+        return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])}
+
+    async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
+        """Rename `artist` to the Patreon display name; the old name when it
+        changed, else None. Unreadable name → no rename, never the handle."""
+        name = await self._platform_display_name("patreon", raw_slug, url)
+        if not name or name == artist.name:
+            return None
+        old = artist.name
+        artist.name = name
+        await self.session.commit()
+        return old
+
+    async def _existing_source(self, platform: str, url: str) -> Source | None:
+        """The source this URL already is, whichever artist owns it. Discord
+        compares ids, not strings, so a row stored before canonicalisation (a
+        ptb host, a trailing slash) is still found."""
+        if platform != DISCORD:
+            return (await self.session.execute(
+                select(Source).where(Source.platform == platform, Source.url == url)
+            )).scalars().first()
+        want = _discord_ids(url)
+        rows = (await self.session.execute(
+            select(Source).where(Source.platform == DISCORD).order_by(Source.id)
+        )).scalars().all()
+        return next((s for s in rows if _discord_ids(s.url) == want), None)
 
     @staticmethod
     def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
@@ -117,8 +320,23 @@ class ExtensionService:
         platforms (and any failure — no credential, network error) fall back to
         the URL handle, which is already readable.
         The resolvers are sync, so they run in an executor."""
+        if platform == DISCORD:
+            # The server's name: what the operator knows the community as.
+            server_id = raw_slug.split("/", 1)[0]
+            names = await self._discord_names(server_id, None)
+            return names.get("server") or f"Discord {server_id}"
+        return await self._platform_display_name(platform, raw_slug, url) or raw_slug
+
+    async def _platform_display_name(
+        self, platform: str, raw_slug: str, url: str
+    ) -> str | None:
+        """The creator's display name as Patreon or SubscribeStar shows it, read
+        with the stored cookies; None when it can't be read (no credential, a
+        network error, a slow answer, any other platform). None, not the handle,
+        so a caller can tell a real name from a fallback — a rename to the
+        Patreon name must never rename to a URL handle instead."""
         if self._crypto is None or platform not in ("patreon", "subscribestar"):
-            return raw_slug
+            return None
         import asyncio
 
         from .credential_service import CredentialService
@@ -128,7 +346,7 @@ class ExtensionService:
             if platform == "patreon":
                 cookies = await cred.get_cookies_path("patreon")
                 from .patreon_resolver import resolve_display_name
-                name = await loop.run_in_executor(
+                call = loop.run_in_executor(
                     None, resolve_display_name, raw_slug,
                     str(cookies) if cookies else None,
                 )
@@ -136,15 +354,14 @@ class ExtensionService:
                 cookies = await cred.get_cookies_path("subscribestar")
                 from .subscribestar_client import SubscribeStarClient
                 client = SubscribeStarClient(str(cookies) if cookies else None)
-                name = await loop.run_in_executor(
-                    None, client.resolve_display_name, url
-                )
+                call = loop.run_in_executor(None, client.resolve_display_name, url)
+            name = await asyncio.wait_for(call, timeout=_NAME_LOOKUP_SECONDS)
         except Exception as exc:      # resolution is best-effort — never block the add
             log.warning("artist display-name resolution failed (%s): %s", platform, exc)
-            return raw_slug
-        return name or raw_slug
+            return None
+        return (name or "").strip() or None
 
-    async def probe(self, url: str) -> dict:
+    async def probe(self, url: str, *, names: bool = False) -> dict:
         """Read-only resolution of a creator-page URL against the FC DB.
         Returns one of:
           - {state: 'unknown_platform'}            — URL didn't match any
@@ -160,21 +377,30 @@ class ExtensionService:
                                                    — exact (artist, platform,
             url) Source already exists
 
-        Side-effect-free: two SELECTs at most.
+        `names` (the Add panel asks, the chip does not) adds `display_name`:
+        the creator's name as Patreon/SubscribeStar shows it, or None. It costs
+        a request to the platform, so a plain page view never pays it.
+
+        Side-effect-free: two SELECTs at most, plus that one lookup.
         """
         try:
             platform, raw_slug = self._derive(url)
         except (UnknownPlatformError, InvalidUrlError):
             return {"state": "unknown_platform"}
+        if platform == DISCORD:
+            return await self._probe_discord(raw_slug)
 
         slug = slugify(raw_slug)
+        result: dict = {"platform": platform, "slug": slug}
+        if names:
+            result["display_name"] = await self._platform_display_name(
+                platform, raw_slug, url,
+            )
         artist = (await self.session.execute(
             select(Artist).where(Artist.slug == slug)
         )).scalar_one_or_none()
         if artist is None:
-            return {"state": "new", "platform": platform, "slug": slug}
-
-        artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
+            return {"state": "new", **result}
 
         source = (await self.session.execute(
             select(Source).where(
@@ -184,26 +410,114 @@ class ExtensionService:
             )
         )).scalar_one_or_none()
         if source is None:
-            return {
-                "state": "artist_match",
-                "platform": platform,
-                "slug": slug,
-                "artist": artist_payload,
-            }
+            return {"state": "artist_match", **result, "artist": self._artist_payload(artist)}
 
         return {
             "state": "source_match",
-            "platform": platform,
-            "slug": slug,
-            "artist": artist_payload,
-            "source": {
-                "id": source.id,
-                "artist_id": source.artist_id,
-                "platform": source.platform,
-                "url": source.url,
-                "enabled": source.enabled,
+            **result,
+            "artist": self._artist_payload(artist),
+            "source": self._source_payload(source),
+        }
+
+    async def _probe_discord(self, raw_slug: str) -> dict:
+        """probe for a Discord server or channel. The states mean what they
+        mean elsewhere, but the artist is never read off the URL:
+
+          - source_match: this channel is a source — or the whole server is
+            (`covered_by_server`), which already walks every channel;
+          - artist_match: another source on this server belongs to an artist,
+            the one this channel most likely belongs to too (a suggestion the
+            Add panel preselects, not a decision);
+          - new: nothing on this server yet.
+
+        `discord` carries the ids, both canonical URLs and the display names,
+        read with the stored token; a name that can't be read is None."""
+        server_id, _, channel_id = raw_slug.partition("/")
+        channel_id = channel_id or None
+        rows = (await self.session.execute(
+            select(Source, Artist)
+            .join(Artist, Artist.id == Source.artist_id)
+            .where(Source.platform == DISCORD)
+            .order_by(Source.id)
+        )).all()
+        exact = server_whole = on_server = None
+        for source, artist in rows:
+            ids = _discord_ids(source.url)
+            if ids is None or ids[0] != server_id:
+                continue
+            if ids[1] == channel_id and exact is None:
+                exact = (source, artist)
+            elif ids[1] is None and server_whole is None:
+                server_whole = (source, artist)
+            if on_server is None:
+                on_server = (source, artist)
+
+        names = await self._discord_names(server_id, channel_id)
+        base = f"https://discord.com/channels/{server_id}"
+        result: dict = {
+            "platform": DISCORD,
+            "slug": raw_slug,
+            "discord": {
+                "server_id": server_id,
+                "channel_id": channel_id,
+                "server_name": names.get("server"),
+                "channel_name": names.get("channel"),
+                "server_url": base,
+                "channel_url": f"{base}/{channel_id}" if channel_id else None,
             },
         }
+        hit = exact or server_whole
+        if hit is not None:
+            source, artist = hit
+            result.update(
+                state="source_match",
+                artist=self._artist_payload(artist),
+                source=self._source_payload(source),
+                covered_by_server=exact is None,
+            )
+        elif on_server is not None:
+            result.update(state="artist_match", artist=self._artist_payload(on_server[1]))
+        else:
+            result["state"] = "new"
+        return result
+
+    async def _discord_names(self, server_id: str | None, channel_id: str | None) -> dict:
+        """Server/channel display names via the stored Discord token. Never
+        raises and never waits out a rate limit: it runs while the operator
+        looks at a page, so a slow or missing answer just means no names."""
+        if self._crypto is None:
+            return {}
+        import asyncio
+
+        from .credential_service import CredentialService
+        from .discord_client import DiscordClient
+        try:
+            token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
+            if not token:
+                return {}
+            client = DiscordClient(token, max_retries=0)
+            loop = asyncio.get_running_loop()
+            return await asyncio.wait_for(
+                loop.run_in_executor(None, client.describe, server_id, channel_id),
+                timeout=_NAME_LOOKUP_SECONDS,
+            )
+        except Exception as exc:      # names are decoration — never fail the call
+            log.info("Discord name lookup failed: %s", exc)
+            return {}
+
+    @staticmethod
+    def _artist_payload(artist) -> dict:
+        return {"id": artist.id, "name": artist.name, "slug": artist.slug}
+
+    @staticmethod
+    def _source_payload(source) -> dict:
+        return {
+            "id": source.id,
+            "artist_id": source.artist_id,
+            "platform": source.platform,
+            "url": source.url,
+            "enabled": source.enabled,
+        }
 
     def _derive(self, url: str) -> tuple[str, str]:
         if not isinstance(url, str) or not url.strip():

backend/app/services/external_fetch.py

@@ -241,11 +241,19 @@ def _fetch_gdrive(url: str, dest_dir: Path, *, read_timeout: float,
 def _fetch_mega(url: str, dest_dir: Path, *, read_timeout: float,
                 total_timeout: float,
                 should_stop: Callable[[], bool]) -> FetchResult:
-    before = set(dest_dir.iterdir()) if dest_dir.exists() else set()
+    # A folder link keeps its subfolders, so files are found at any depth.
+    # Dotfiles are megatools' `.megatmp.*` staging names.
+    def _complete_files() -> set[Path]:
+        return {
+            p for p in dest_dir.rglob("*")
+            if p.is_file() and not p.name.startswith(".")
+        } if dest_dir.exists() else set()
+
+    before = _complete_files()
     # megatools is a subprocess: its timeout IS a total wall-clock cap (the read
     # timeout has no analogue here), so the total budget applies directly.
     _run_mega_get(url, str(dest_dir), timeout=total_timeout)
-    new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()]
+    new = sorted(_complete_files() - before)
     if not new:
         return FetchResult(error="mega-get wrote no new file")
     return FetchResult(files=new, bytes=sum(p.stat().st_size for p in new))

backend/app/services/external_links.py

@@ -21,9 +21,9 @@ log = logging.getLogger(__name__)
 
 
 def refetch_links_for_post(session: Session, post_id: int) -> dict:
-    """Reset every settled ExternalLink on a post to pending (fresh attempt
-    budget) and dispatch its fetch. In-flight ('downloading') links are left
-    alone. Commits. Returns {links_total, links_reset}."""
+    """Reset every settled ExternalLink on a post to pending (fresh attempt and
+    recovery budgets) and dispatch its fetch. In-flight ('downloading') links
+    are left alone. Commits. Returns {links_total, links_reset}."""
     links = session.execute(
         select(ExternalLink).where(ExternalLink.post_id == post_id)
     ).scalars().all()
@@ -33,6 +33,7 @@ def refetch_links_for_post(session: Session, post_id: int) -> dict:
             continue
         link.status = "pending"
         link.attempts = 0
+        link.recoveries = 0
         link.last_error = None
         link.completed_at = None
         reset_ids.append(link.id)

backend/app/services/gallery_dl.py

@@ -6,6 +6,9 @@ backend/app/services/gallery_dl.py). FC adaptations:
   - Validation uses backend/app/services/file_validator.py
   - Config dir under `<images_root>/.gallery-dl/`, not a separate config_path
   - PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b
+
+The result it returns (`DownloadResult`, `ErrorType`) is shared with the native
+ingesters and lives in download_result.
 """
 
 from __future__ import annotations
@@ -19,39 +22,20 @@ import tempfile
 import time
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
-from enum import StrEnum
 from pathlib import Path
 
+from .download_result import (
+    DownloadResult,
+    ErrorType,
+    classify_tier_gated,
+    make_run_stats,
+    tier_gated_message,
+)
 from .file_validator import is_validatable, quarantine_file, validate_file
 
 log = logging.getLogger(__name__)
 
 
-class ErrorType(StrEnum):
-    AUTH_ERROR = "auth_error"
-    RATE_LIMITED = "rate_limited"
-    NOT_FOUND = "not_found"
-    ACCESS_DENIED = "access_denied"
-    NETWORK_ERROR = "network_error"
-    NO_NEW_CONTENT = "no_new_content"
-    TIER_LIMITED = "tier_limited"
-    TIMEOUT = "timeout"
-    HTTP_ERROR = "http_error"
-    UNSUPPORTED_URL = "unsupported_url"
-    VALIDATION_FAILED = "validation_failed"
-    # Native Patreon ingester only (plan #697): a response parsed as JSON but
-    # didn't match the JSON:API shape the ingester depends on (missing `data`,
-    # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
-    # is updating the ingester's field-set/parser, not rotating credentials. The
-    # contract test guards against silently shipping this.
-    API_DRIFT = "api_drift"
-    # Run made real progress (downloaded ≥1 file) but did not finish in the
-    # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
-    # mapping classifies this as "ok" because the next tick continues.
-    PARTIAL = "partial"
-    UNKNOWN_ERROR = "unknown_error"
-
-
 # Tick mode (routine cron polls): skip ≤20 contiguous already-archived
 # items, then exit gallery-dl. Established subscription with zero new
 # content exits in ~30s of HEAD requests instead of walking to the bottom
@@ -94,48 +78,6 @@ BACKFILL_CHUNK_SECONDS = 600
 _DEFAULT_GDL_TIMEOUT_SECONDS = 870
 
 
-# --- Discord naming ---------------------------------------------------------
-#
-# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
-# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
-# for an attachment: `channel` is a plain STRING (the channel's name), the
-# message is `message_id`, the attachment's position in it is `num`, and there
-# is NO `id` key at all.
-#
-# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
-# "None", so every Discord download since the platform was added landed in a
-# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
-# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
-# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
-# got a Post or a post date, and (b) collides: every `image.png` in a channel
-# overwrote the same `image.json`, so the one sidecar that survived described
-# whichever message happened to be written last.
-#
-# The fix names the sidecar EXACTLY like the media minus its extension, so
-# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
-# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
-# these patterns against a sanitized copy of the real sidecar, so a key that
-# does not exist fails CI instead of silently becoming "None".
-DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
-DISCORD_DIRECTORY = ["{channel}"]
-
-
-def sidecar_name_for(media_pattern: str) -> str | None:
-    """The metadata filename pattern that names a sidecar exactly like its media.
-
-    Returns None for a pattern that does not end in `.{extension}`, since then
-    there is no media stem to mirror and the caller must fall back.
-    """
-    suffix = ".{extension}"
-    if not media_pattern.endswith(suffix):
-        return None
-    return media_pattern[: -len(suffix)] + ".json"
-
-
-def metadata_postprocessor(filename: str) -> dict:
-    return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
-
-
 def archive_path(images_root: Path) -> Path:
     """gallery-dl's download archive: the record of what it has already fetched.
 
@@ -189,82 +131,6 @@ class SourceConfig:
         )
 
 
-@dataclass
-class DownloadResult:
-    success: bool
-    url: str
-    artist_slug: str
-    platform: str
-    files_downloaded: int = 0
-    files_quarantined: int = 0
-    quarantined_paths: list[str] = field(default_factory=list)
-    written_paths: list[str] = field(default_factory=list)
-    # Native ingester only: post-only sidecar paths for posts that have NO
-    # downloadable media (pure-text posts), so the importer can still upsert the
-    # Post + its body. Empty on the gallery-dl path. Phase 3 imports these via
-    # Importer.upsert_post_record (keyed on external_post_id → updates, never
-    # doubles, the same Post a media import would create).
-    post_record_paths: list[str] = field(default_factory=list)
-    # Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
-    # pairs for already-present media whose ImageRecord.source_filehash should be
-    # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
-    # on the gallery-dl path and outside recapture.
-    relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
-    stdout: str = ""
-    stderr: str = ""
-    return_code: int = 0
-    error_type: ErrorType | None = None
-    error_message: str | None = None
-    duration_seconds: float = 0.0
-    started_at: str | None = None
-    completed_at: str | None = None
-    # Plan #704 — structured fields the NATIVE ingester populates directly (None
-    # on the gallery-dl path, which keeps the regex-over-stdout route). When set,
-    # phase 3 reads these instead of scraping the text it would otherwise have to
-    # reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the
-    # backfill checkpoint the ingester knows exactly (no parse_last_cursor).
-    run_stats: dict | None = None
-    cursor: str | None = None
-    posts_processed: int = 0
-    # Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so
-    # the platform cooldown matches the hint instead of a flat default. None when
-    # unknown (no header, or not a rate-limit failure).
-    retry_after_seconds: float | None = None
-
-
-def extract_errors_warnings(stderr: str) -> str:
-    """Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
-
-    A generic download-log helper (module-level so the native-ingester result
-    path can shape its logs without reaching through a GalleryDLService instance).
-    """
-    if not stderr:
-        return ""
-    kept = [
-        line for line in stderr.splitlines()
-        if "][error]" in line.lower() or "][warning]" in line.lower()
-    ]
-    return "\n".join(kept)
-
-
-def truncate_log(text: str, max_bytes: int = 500_000) -> str:
-    """Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
-    if not text:
-        return text
-    encoded = text.encode("utf-8")
-    if len(encoded) <= max_bytes:
-        return text
-    half = max_bytes // 2
-    head = encoded[:half].decode("utf-8", errors="ignore")
-    tail = encoded[-half:].decode("utf-8", errors="ignore")
-    head_lines = head.count("\n")
-    tail_lines = tail.count("\n")
-    total_lines = text.count("\n")
-    elided = max(0, total_lines - head_lines - tail_lines)
-    marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
-    return head + marker + tail
-
-
 def _summarize_validation_failures(failures: list[dict]) -> str:
     if not failures:
         return "Validation failed"
@@ -279,106 +145,6 @@ def _summarize_validation_failures(failures: list[dict]) -> str:
     return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
 
 
-# (parse_last_cursor was removed in plan #704: the native ingester now carries
-# its checkpoint cursor as a structured DownloadResult.cursor field, so there is
-# no log text to scrape — and gallery-dl platforms never had a cursor.)
-
-
-def make_run_stats(
-    *,
-    exit_code: int = 0,
-    downloaded_count: int = 0,
-    skipped_count: int = 0,
-    per_item_failures: int = 0,
-    warning_count: int = 0,
-    tier_gated_count: int = 0,
-    quarantined_count: int = 0,
-    dead_lettered_count: int = 0,
-) -> dict:
-    """The canonical `run_stats` dict shape, in ONE place.
-
-    Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
-    native ingester's per-outcome tally (ingest_core) — build through this so the
-    key set can't drift between backends. Phase 3 + the Logs UI read these keys.
-    """
-    return {
-        "exit_code": exit_code,
-        "downloaded_count": downloaded_count,
-        "skipped_count": skipped_count,
-        "per_item_failures": per_item_failures,
-        "warning_count": warning_count,
-        "tier_gated_count": tier_gated_count,
-        "quarantined_count": quarantined_count,
-        "dead_lettered_count": dead_lettered_count,
-    }
-
-
-# --- tier-gated classification, shared by BOTH backends ---------------------
-#
-# These three live together because the native ingester and the gallery-dl
-# subprocess must reach the same verdict from the same number. They did not:
-# gallery-dl classified TIER_LIMITED while ingest_core counted gated posts and
-# threw the count away, so the platforms FC owns reported a paywalled creator as
-# a silent one (#874 follow-up). One predicate, spread into both, rather than
-# the condition re-derived per backend.
-
-
-def classify_tier_gated(tier_gated_count: int) -> ErrorType | None:
-    """TIER_LIMITED when a walk saw tier-gated posts and nothing else failed.
-
-    Deliberately NOT conditioned on `downloaded == 0`. A creator whose top-tier
-    posts we cannot see is tier-limited even in a week we did get their cheaper
-    ones — the fact the operator needs ("there is content here you are not
-    paying for") is true either way. gallery-dl has classified it this way since
-    the paywall-as-"needs attention" complaint (see `_categorize_error`), and the
-    native path now matches rather than inventing a stricter rule.
-
-    Callers must apply this only AFTER the real error categories (auth, rate
-    limit, drift, …) have had their turn; tier-gating is the weakest signal and
-    must never mask a genuine failure.
-    """
-    return ErrorType.TIER_LIMITED if tier_gated_count else None
-
-
-def tier_gated_message(count: int) -> str:
-    """The one wording for the tier-gated verdict, so the two backends can't
-    describe the same state differently in the Logs UI."""
-    return (
-        f"Subscription tier does not grant access to "
-        f"{count} post{'s' if count != 1 else ''}"
-    )
-
-
-# `Source.error_type` doubles as the failure-class chip, and a status of "ok"
-# CLEARS it (alembic 0032). TIER_LIMITED breaks that assumption: it rides an
-# otherwise-successful run, so without an exemption the chip is wiped the moment
-# it is set and `FailingSourcesCard`'s `tier_limited` palette entry can never
-# render. Informational classes are the exemption — they describe the source,
-# not a failure of the run.
-INFORMATIONAL_ERROR_TYPES = frozenset({ErrorType.TIER_LIMITED.value})
-
-
-def is_informational(error_type) -> bool:
-    """True for a class that reports a state rather than a failure. Accepts an
-    ErrorType or the plain string persisted on Source.error_type."""
-    return error_type is not None and str(error_type) in INFORMATIONAL_ERROR_TYPES
-
-
-def walk_completed(result: DownloadResult) -> bool:
-    """Did this walk reach the bottom cleanly?
-
-    The backfill lifecycle's completion test. An informational error_type still
-    counts as complete: a fully-paywalled creator's backfill DID finish, and
-    treating it as unfinished re-walks the same wall until the stall counter
-    trips — the creator we can see least becoming the one we fetch most.
-    """
-    return (
-        result.success
-        and result.return_code == 0
-        and (result.error_type is None or is_informational(result.error_type))
-    )
-
-
 class GalleryDLService:
     """Service for executing gallery-dl downloads."""
 
@@ -417,27 +183,16 @@ class GalleryDLService:
     # (services/patreon_ingester.py), not gallery-dl.
     PLATFORM_DEFAULTS = {
         # subscribestar removed — native-ingester platform now (#71); pixiv
-        # removed likewise (#129); deviantart removed at #3069 as a dropped
-        # platform, not a migrated one. The remaining entries are the
-        # gallery-dl platforms not yet migrated.
+        # removed likewise (#129); discord likewise (milestone 428, whose
+        # downloader keeps this config's on-disk naming); deviantart removed at
+        # #3069 as a dropped platform, not a migrated one. HentaiFoundry is the
+        # one platform left here, by the operator's choice not to migrate it.
         "hentaifoundry": {
             "content_types": ["all"],
             "directory": [],
             "filename": "{category}_{index:>03}_{title[:50]}.{extension}",
             "include": "all",
         },
-        "discord": {
-            "content_types": ["all"],
-            "directory": DISCORD_DIRECTORY,
-            "filename": DISCORD_FILENAME,
-            # Overrides the global `{filename}.json` sidecar for this extractor
-            # only — see the Discord naming note above.
-            "postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
-            "embeds": "all",
-            "stickers": True,
-            "reactions": False,
-            "threads": True,
-        },
     }
 
     def __init__(
@@ -552,17 +307,6 @@ class GalleryDLService:
         if source_config.filename_pattern:
             platform_section["filename"] = source_config.filename_pattern
 
-        # A platform that names its sidecar after its media must keep doing so
-        # under a per-source filename override, or the pairing breaks exactly the
-        # way Discord's did. No metadata wanted means no platform postprocessor
-        # either — the global list was already dropped above.
-        if "postprocessors" in platform_section:
-            mirrored = sidecar_name_for(platform_section.get("filename") or "")
-            if not source_config.save_metadata or mirrored is None:
-                platform_section.pop("postprocessors")
-            else:
-                platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
-
         platform_section["metadata"] = source_config.save_metadata
 
         return config
@@ -778,16 +522,6 @@ class GalleryDLService:
             tier_gated_count=tier_gated_count,
         )
 
-    @staticmethod
-    def _extract_errors_warnings(stderr: str) -> str:
-        # Thin delegator to the module-level helper (kept for existing callers).
-        return extract_errors_warnings(stderr)
-
-    @staticmethod
-    def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
-        # Thin delegator to the module-level helper (kept for existing callers).
-        return truncate_log(text, max_bytes)
-
     async def download(
         self,
         url: str,
@@ -810,9 +544,6 @@ class GalleryDLService:
 
         if cookies_path:
             config["extractor"]["cookies"] = cookies_path
-        if auth_token and platform == "discord":
-            config["extractor"].setdefault("discord", {})
-            config["extractor"]["discord"]["token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
@@ -996,8 +727,6 @@ class GalleryDLService:
         config = self._build_config_for_source(platform, source_config, artist_slug)
         if cookies_path:
             config["extractor"]["cookies"] = cookies_path
-        if auth_token and platform == "discord":
-            config["extractor"].setdefault("discord", {})["token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),

backend/app/services/importer.py

@@ -16,6 +16,7 @@ from dataclasses import dataclass, field
 from enum import StrEnum
 from pathlib import Path
 
+from celery.exceptions import SoftTimeLimitExceeded
 from PIL import Image
 from sqlalchemy import func, select, update
 from sqlalchemy.exc import IntegrityError
@@ -43,16 +44,15 @@ from ..utils.paths import (
 from ..utils.phash import compute_phash, find_similar, fingerprint_path, fingerprints_match
 from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
-from .archive_extractor import extract_archive, is_archive
+from ..utils.staging import is_temp_staging_file
+from .archive_extractor import extract_archive, is_archive, software_marker
 from .attachment_store import AttachmentStore
 from .audits import single_color
 from .link_extract import extract_external_links
 from .thumbnailer import Thumbnailer
 from .wip_title import (
-    WIP_TITLE_SOFT_SOURCE,
     WIP_TITLE_SOURCE,
     apply_wip_image_tags,
-    matches_soft_wip_title,
     matches_wip_title,
     resolve_wip_tag_id,
 )
@@ -117,6 +117,21 @@ def is_video(path: Path) -> bool:
     return path.suffix.lower() in VIDEO_EXTS
 
 
+def _not_content(path: Path) -> ImportResult | None:
+    """The skip result for a file that is never library content: a sidecar
+    `.json` (metadata) or a downloader's staging file (an unfinished download,
+    which becomes content only under its final name). None for anything else."""
+    if path.suffix.lower() == ".json":
+        error = "sidecar json is metadata, not content"
+    elif is_temp_staging_file(path):
+        error = "unfinished download (staging file), not content"
+    else:
+        return None
+    return ImportResult(
+        status="skipped", skip_reason=SkipReason.invalid_image, error=error,
+    )
+
+
 def _safe_ext(path: Path) -> str:
     """Conservatively extract a file extension for PostAttachment.ext
     (varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for
@@ -293,6 +308,44 @@ class Importer:
             sp.rollback()
             return self.session.execute(stmt).scalar_one()
 
+    def _store_new_record(self, record: ImageRecord) -> ImageRecord | None:
+        """Insert `record`, or return the row that already holds its bytes.
+
+        The sha256 lookup before an insert can lose a race: two copies of one
+        picture in different folders, imported side by side by two workers,
+        both find no row, and the unique index refuses the second insert. Only
+        that insert is undone (a savepoint), so the session stays usable and
+        the caller files its file as a duplicate of the winner. None means
+        `record` was stored."""
+        sp = self.session.begin_nested()
+        try:
+            self.session.add(record)
+            self.session.flush()
+            sp.commit()
+            return None
+        except IntegrityError:
+            sp.rollback()
+            winner = self.session.execute(
+                select(ImageRecord).where(ImageRecord.sha256 == record.sha256)
+            ).scalar_one_or_none()
+            if winner is None:
+                raise
+            return winner
+
+    def _file_as_duplicate(
+        self, existing: ImageRecord, path: Path, artist: Artist | None,
+        explicit_source: Source | None,
+    ) -> ImportResult:
+        """A byte-identical copy of `existing`: link the copy's post to the
+        image (enrich-on-duplicate), so a picture posted twice shows on both
+        posts, and report the skip."""
+        self._apply_sidecar(existing, path, artist, explicit_source=explicit_source)
+        self.session.commit()
+        return ImportResult(
+            status="skipped", skip_reason=SkipReason.duplicate_hash,
+            image_id=existing.id, error="sha256 already present",
+        )
+
     def _find_or_create_source(
         self, *, artist_id: int, platform: str, url: str,
     ) -> Source:
@@ -412,11 +465,8 @@ class Importer:
         media members (one Post via the archive-adjacent sidecar) and
         preserve the archive itself. Other non-media → PostAttachment.
         Nothing is skipped for being non-media (FC-2d-iii)."""
-        if source.suffix.lower() == ".json":
-            return ImportResult(
-                status="skipped", skip_reason=SkipReason.invalid_image,
-                error="sidecar json is metadata, not content",
-            )
+        if (refused := _not_content(source)) is not None:
+            return refused
         if is_archive(source):
             return self._import_archive(source)
         if not is_supported(source):
@@ -428,11 +478,19 @@ class Importer:
         return self._upsert_artist(name) if name else None
 
     def _post_for_sidecar(
-        self, source: Path, artist: Artist | None
+        self, source: Path, artist: Artist | None,
+        *, source_row: Source | None = None,
     ) -> Post | None:
         """If a sidecar sits next to `source`, ensure its Source+Post
         exist (idempotent) and return the Post — so attachments can link
-        to the same Post the per-member _apply_sidecar will reuse."""
+        to the same Post the per-member _apply_sidecar will reuse.
+
+        `source_row` is the subscription being downloaded, when there is one,
+        and wins over the (artist, platform) lookup — as it does in
+        `upsert_post_record`. The lookup takes the artist's FIRST source on the
+        platform, which is right only while an artist has one: a Discord artist
+        has one per channel, and every non-image file from a later channel was
+        filed under the first as an undated second post (#4435)."""
         sc = find_sidecar(source)
         if sc is None or artist is None:
             return None
@@ -444,10 +502,13 @@ class Importer:
             log.warning("sidecar parse failed for %s: %s", sc, exc)
             return None
         sd = parse_sidecar(data)
-        platform = sd.platform or "unknown"
-        src = self._lookup_source_for_sidecar(
-            artist_id=artist.id, platform=platform,
-        )
+        if source_row is not None:
+            src = source_row
+        else:
+            platform = sd.platform or "unknown"
+            src = self._lookup_source_for_sidecar(
+                artist_id=artist.id, platform=platform,
+            )
         epid = sd.external_post_id or sc.stem
         return self._find_or_create_post(
             source_id=src.id if src else None,
@@ -533,28 +594,42 @@ class Importer:
         # lays files out under /images/<artist_slug>/...). Filesystem
         # import still calls bare _import_archive(source) and falls
         # back to the path-walk derivation as before.
-        probe = safe_probe.probe_archive(source)
+        #
+        # A game or app build is recognised first, from the archive's directory
+        # alone. The probe reads and checksums every member, which for a
+        # multi-gigabyte build is minutes of work on a file that is kept whole
+        # and never extracted anyway.
+        marker = software_marker(source)
+        probe = None if marker is not None else safe_probe.probe_archive(source)
+        if probe is not None and probe.crashed:
+            return ImportResult(
+                status="failed",
+                error=f"archive probe crashed/timed out: {probe.reason}",
+            )
+
+        artist_use = artist if artist is not None else self._resolve_artist(source)
+        post = self._post_for_sidecar(source, artist_use, source_row=source_row)
+        if marker is not None:
+            # A game or app build: kept whole, its bundled images left alone.
+            self._capture_attachment(
+                source, post=post, artist=artist_use, resolved=True,
+            )
+            log.info("%s: a software build (holds %s), kept unextracted",
+                     source.name, marker)
+            return ImportResult(status="attached")
         if not probe.ok:
-            if probe.crashed:
-                return ImportResult(
-                    status="failed",
-                    error=f"archive probe crashed/timed out: {probe.reason}",
-                )
-            # Clean rejection (bomb cap exceeded, integrity mismatch):
-            # still preserve the archive file itself as an attachment so
-            # nothing silently vanishes, matching extract_archive's
-            # fail-soft contract.
-            artist_use = artist if artist is not None else self._resolve_artist(source)
-            post = self._post_for_sidecar(source, artist_use)
+            # Clean rejection (bomb cap exceeded, integrity mismatch): the
+            # archive file itself is still kept as an attachment so nothing
+            # silently vanishes, matching extract_archive's fail-soft contract.
             self._capture_attachment(
                 source, post=post, artist=artist_use, resolved=True,
             )
             reason = f"archive probe rejected, captured unextracted: {probe.reason}"
             log.warning("%s: %s", source.name, reason)
             return ImportResult(status="attached", error=reason)
-
-        artist_use = artist if artist is not None else self._resolve_artist(source)
-        post = self._post_for_sidecar(source, artist_use)
+        # Committed before extraction, so a member that fails (and is rolled
+        # back on its own) can never take the archive's post with it.
+        self.session.commit()
         member_ids: list[int] = []
         # Every member image touched (new + superseded + deduped), so the
         # from_attachment_id stamp below covers files that already existed in the
@@ -567,7 +642,7 @@ class Importer:
         # or failed/corrupt members — instead of one catch-all string.
         counts = {
             "media": 0, "deduped": 0, "unsupported": 0,
-            "failed": 0, "nested": 0, "nested_rejected": 0,
+            "failed": 0, "nested": 0, "nested_rejected": 0, "nested_software": 0,
         }
         self._collect_archive_members(
             source, attribution=source, source_row=source_row,
@@ -599,20 +674,21 @@ class Importer:
         if counts["media"] == 0 and counts["nested"] == 0:
             reason = ("archive yielded no members (unsupported/corrupt, or the "
                       "extractor backend failed)")
-        elif counts["deduped"] and not (
+        elif (counts["deduped"] or counts["nested_software"]) and not (
             counts["unsupported"] or counts["failed"] or counts["nested_rejected"]
         ):
             log.info(
-                "%s: all %d image member(s) already in library (deduped + linked "
-                "to this post) — archive preserved, no new import",
-                source.name, counts["deduped"],
+                "%s: no new images — %d already in the library (linked to this "
+                "post), %d nested software build(s) skipped; archive preserved",
+                source.name, counts["deduped"], counts["nested_software"],
             )
             return ImportResult(status="attached")
         else:
             reason = (
                 f"no new images — media={counts['media']} deduped={counts['deduped']} "
                 f"unsupported/non-media={counts['unsupported']} failed={counts['failed']} "
-                f"nested={counts['nested']} nested-rejected={counts['nested_rejected']}"
+                f"nested={counts['nested']} nested-rejected={counts['nested_rejected']} "
+                f"nested-software={counts['nested_software']}"
             )
         log.warning("%s: %s", source.name, reason)
         return ImportResult(status="attached", error=reason)
@@ -645,6 +721,15 @@ class Importer:
                                 _ARCHIVE_MAX_DEPTH, member_path.name,
                             )
                             continue
+                        # Recognised from its directory before the probe reads
+                        # every member, as for the outer archive.
+                        if (marker := software_marker(member_path)) is not None:
+                            counts["nested_software"] += 1
+                            log.info(
+                                "nested archive is a software build (holds %s), "
+                                "skipped: %s", marker, member_path.name,
+                            )
+                            continue
                         probe = safe_probe.probe_archive(member_path)
                         if not probe.ok:
                             counts["nested_rejected"] += 1
@@ -681,7 +766,15 @@ class Importer:
                             member_record_ids.add(res.image_id)
                     else:
                         counts["failed"] += 1
+        except SoftTimeLimitExceeded:
+            # The task's time budget is spent: stop here so the task marks
+            # its row failed, rather than carrying on until the hard kill.
+            raise
         except Exception as exc:  # noqa: BLE001 — defensive per level; keep going
+            # Each member commits on its own, so this undoes only the one in
+            # progress; without it the next query fails on the dead transaction
+            # and the archive is never stored.
+            self.session.rollback()
             log.warning(
                 "archive extraction failed for %s (depth %d): %s",
                 archive_path.name, depth, exc,
@@ -781,7 +874,6 @@ class Importer:
         # Compute file dimensions (images only) and apply filters.
         width = height = None
         duration = None  # video container duration (Tier-1 near-dup key, #871)
-        has_alpha = False
         if is_video(source):
             # Layer-3 isolation: validate the container via ffprobe (a
             # separate process) before the rest of the pipeline touches
@@ -800,52 +892,9 @@ class Importer:
                 )
             width, height, duration = probe.width, probe.height, probe.duration
         else:
-            try:
-                with Image.open(source) as im:
-                    im.verify()
-                with Image.open(source) as im:
-                    width, height = im.size
-                    has_alpha = im.mode in ("RGBA", "LA") or (
-                        im.mode == "P" and "transparency" in im.info
-                    )
-            except Exception as exc:
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.invalid_image, error=str(exc),
-                )
-
-            if (self.settings.min_width and width < self.settings.min_width) or (
-                self.settings.min_height and height < self.settings.min_height
-            ):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.too_small,
-                    error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
-                )
-
-            if self.settings.skip_transparent and has_alpha:
-                try:
-                    pct = self._transparency_pct(source)
-                except OSError as exc:
-                    # PIL.verify() at line 263 only validates header structure;
-                    # truncated/corrupt pixel data only surfaces when load()
-                    # actually decodes (here via getchannel('A')). Convert to
-                    # invalid_image skip so the Celery autoretry loop doesn't
-                    # bounce the same broken file forever.
-                    return ImportResult(
-                        status="skipped", skip_reason=SkipReason.invalid_image,
-                        error=f"PIL load failed during transparency check: {exc}",
-                    )
-                if pct >= self.settings.transparency_threshold:
-                    return ImportResult(
-                        status="skipped", skip_reason=SkipReason.too_transparent,
-                        error=f"{pct:.2%} transparent",
-                    )
-
-            if self.settings.skip_single_color and self._single_color_hit(source):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.single_color,
-                    error=(f"one color dominates >"
-                           f"{self.settings.single_color_threshold:.0%}"),
-                )
+            refusal, width, height = self._screen_still(source)
+            if refusal is not None:
+                return refusal
 
         # Artist anchored to the attribution path (folder→artist), resolved
         # UP-FRONT so the enrich-on-duplicate branches link provenance with the
@@ -862,34 +911,17 @@ class Importer:
         existing = self.session.execute(existing_stmt).scalar_one_or_none()
         if existing:
             if not self.deep:
-                # Enrich-on-duplicate (parity with attach_in_place): a re-scanned
-                # file that is a byte-dup of an existing image still links its
-                # post via _apply_sidecar, so a cross-posted image shows on every
-                # post.
-                self._apply_sidecar(
-                    existing, attribution_path, path_artist,
-                    explicit_source=explicit_source,
-                )
-                self.session.commit()
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.duplicate_hash,
-                    image_id=existing.id, error="sha256 already present",
+                return self._file_as_duplicate(
+                    existing, attribution_path, path_artist, explicit_source,
                 )
             return self._deep_rederive(existing, source, attribution_path)
 
         # Perceptual near-dup (images only; videos keep phash NULL).
         phash = None
         if not is_video(source):
-            try:
-                with Image.open(source) as im:
-                    phash = compute_phash(im)
-            except OSError as exc:
-                # Same rationale as the transparency-check guard above:
-                # broken-pixel-data files pass verify() but blow up here.
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.invalid_image,
-                    error=f"PIL load failed during phash compute: {exc}",
-                )
+            refusal, phash = self._phash_of(source)
+            if refusal is not None:
+                return refusal
             if phash is not None:
                 candidates = self._phash_candidates_cache()
                 rel, match_id = find_similar(
@@ -959,8 +991,14 @@ class Importer:
             origin="imported_filesystem",
             integrity_status="unknown",
         )
-        self.session.add(record)
-        self.session.flush()
+        if (winner := self._store_new_record(record)) is not None:
+            # The copy just written is a stray unless it is the winner's own
+            # file (the same source imported twice lands on the same path).
+            if Path(winner.path) != dest:
+                dest.unlink(missing_ok=True)
+            return self._file_as_duplicate(
+                winner, attribution_path, path_artist, explicit_source,
+            )
         self._phash_cache_append(phash, width, height, record.id)
 
         # Folder→artist (path_artist resolved up-front above); bind it to the
@@ -1040,9 +1078,7 @@ class Importer:
         removal sticks. The existing catalogue is covered separately by the
         operator-triggered backfill sweep. Gated by the settings toggle, and
         best-effort: any failure is logged, never allowed to fail the import."""
-        hard_on = self.settings.wip_title_tagging_enabled
-        soft_on = self.settings.wip_soft_title_tagging_enabled
-        if not (hard_on or soft_on):
+        if not self.settings.wip_title_tagging_enabled:
             return
         if record.primary_post_id is None:
             return
@@ -1050,21 +1086,14 @@ class Importer:
             title = self.session.execute(
                 select(Post.post_title).where(Post.id == record.primary_post_id)
             ).scalar_one_or_none()
-            # HARD tier ("WIP"/"work in progress") wins — higher precision, and it
-            # trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
-            # that never trains (source wip_title_soft).
-            if hard_on and matches_wip_title(title):
-                source = WIP_TITLE_SOURCE
-            elif soft_on and matches_soft_wip_title(title):
-                source = WIP_TITLE_SOFT_SOURCE
-            else:
+            if not matches_wip_title(title):
                 return
             if self._wip_tag_id is _UNSET:
                 self._wip_tag_id = resolve_wip_tag_id(self.session)
             if self._wip_tag_id is None:
                 return
             apply_wip_image_tags(
-                self.session, [record.id], self._wip_tag_id, source=source
+                self.session, [record.id], self._wip_tag_id, source=WIP_TITLE_SOURCE
             )
         except Exception as exc:  # noqa: BLE001 — a tag must never fail an import
             log.warning(
@@ -1141,9 +1170,51 @@ class Importer:
         if post.artist_id is None:
             post.artist_id = artist.id
         self._apply_post_fields(post, sd)
+        self._redate_post_images(post)
         self.session.commit()
         return True
 
+    def _redate_post_images(self, post: Post) -> None:
+        """Carry a post's date onto the images already linked to it (#4431).
+
+        The native ingesters import a post's media BEFORE its record: the
+        per-media sidecar holds only the image identity (post-first, #856), and
+        the date arrives with `_post.json`. So `_attach_provenance` links each
+        image to a post that has no date yet, and the image keeps its download
+        time. This runs when the record lands, and applies the same two rules
+        `_attach_provenance` applies: `effective_date` is the PRIMARY post's
+        date, and `earliest_post_date` is the earliest date across every post
+        the image is linked to. Only rows that differ are written."""
+        if post.post_date is None:
+            return
+        self.session.flush()
+        self.session.execute(
+            update(ImageRecord)
+            .where(ImageRecord.primary_post_id == post.id)
+            .where(ImageRecord.effective_date.is_distinct_from(post.post_date))
+            .values(effective_date=post.post_date)
+            .execution_options(synchronize_session=False)
+        )
+        linked = select(ImageProvenance.image_record_id).where(
+            ImageProvenance.post_id == post.id
+        )
+        earliest = (
+            select(func.min(Post.post_date))
+            .select_from(ImageProvenance)
+            .join(Post, Post.id == ImageProvenance.post_id)
+            .where(ImageProvenance.image_record_id == ImageRecord.id)
+            .where(Post.post_date.is_not(None))
+            .correlate(ImageRecord)
+            .scalar_subquery()
+        )
+        self.session.execute(
+            update(ImageRecord)
+            .where(ImageRecord.id.in_(linked))
+            .where(ImageRecord.earliest_post_date.is_distinct_from(earliest))
+            .values(earliest_post_date=earliest)
+            .execution_options(synchronize_session=False)
+        )
+
     def attach_in_place(
         self,
         path: Path,
@@ -1177,62 +1248,28 @@ class Importer:
             caller may delete the on-disk original (mirrors duplicate_hash)
           - failed → file untouched; caller decides
         """
-        if path.suffix.lower() == ".json":
-            return ImportResult(
-                status="skipped", skip_reason=SkipReason.invalid_image,
-                error="sidecar json is metadata, not content",
-            )
+        if (refused := _not_content(path)) is not None:
+            return refused
         if is_archive(path):
             return self._import_archive(
                 path, artist=artist, source_row=source,
             )
         if not is_supported(path):
-            post = self._post_for_sidecar(path, artist) if artist else None
+            post = (
+                self._post_for_sidecar(path, artist, source_row=source)
+                if artist else None
+            )
             return self._capture_attachment(
                 path, post=post, artist=artist, resolved=True,
             )
 
-        # Format / dimension / transparency filters (mirror _import_media).
+        # The same content filters as _import_media.
         width = height = None
         duration = None  # video container duration (Tier-1 near-dup key, #871)
-        has_alpha = False
         if not is_video(path):
-            try:
-                with Image.open(path) as im:
-                    im.verify()
-                with Image.open(path) as im:
-                    width, height = im.size
-                    has_alpha = im.mode in ("RGBA", "LA") or (
-                        im.mode == "P" and "transparency" in im.info
-                    )
-            except Exception as exc:
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.invalid_image,
-                    error=str(exc),
-                )
-
-            if (self.settings.min_width and width < self.settings.min_width) or (
-                self.settings.min_height and height < self.settings.min_height
-            ):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.too_small,
-                    error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
-                )
-
-            if self.settings.skip_transparent and has_alpha:
-                pct = self._transparency_pct(path)
-                if pct >= self.settings.transparency_threshold:
-                    return ImportResult(
-                        status="skipped", skip_reason=SkipReason.too_transparent,
-                        error=f"{pct:.2%} transparent",
-                    )
-
-            if self.settings.skip_single_color and self._single_color_hit(path):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.single_color,
-                    error=(f"one color dominates >"
-                           f"{self.settings.single_color_threshold:.0%}"),
-                )
+            refusal, width, height = self._screen_still(path)
+            if refusal is not None:
+                return refusal
         else:
             # Best-effort probe for dims + duration so downloaded videos can dedup
             # (#871). LENIENT: unlike _import_media this path does not reject on a
@@ -1255,21 +1292,14 @@ class Importer:
             # only sets it when NULL. Without this the new post is left with no
             # image AND (if its other content also deduped) no attachment, i.e. a
             # bare shell — operator-flagged 2026-06-08 (1589 empty Anduo posts).
-            self._apply_sidecar(existing, path, artist, explicit_source=source)
-            self.session.commit()
-            return ImportResult(
-                status="skipped", skip_reason=SkipReason.duplicate_hash,
-                image_id=existing.id, error="sha256 already present",
-            )
+            return self._file_as_duplicate(existing, path, artist, source)
 
         # phash dedup + supersede
         phash = None
         if not is_video(path):
-            try:
-                with Image.open(path) as im:
-                    phash = compute_phash(im)
-            except Exception:
-                phash = None
+            refusal, phash = self._phash_of(path)
+            if refusal is not None:
+                return refusal
             if phash is not None:
                 candidates = self._phash_candidates_cache()
                 rel, match_id = find_similar(
@@ -1340,8 +1370,8 @@ class Importer:
         )
         if artist is not None:
             record.artist_id = artist.id
-        self.session.add(record)
-        self.session.flush()
+        if (winner := self._store_new_record(record)) is not None:
+            return self._file_as_duplicate(winner, path, artist, source)
         self._phash_cache_append(phash, width, height, record.id)
 
         # Sidecar provenance (best-effort). When `source` is passed, link
@@ -1356,6 +1386,51 @@ class Importer:
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
+    def discard_after_attach(self, path: Path, result: ImportResult) -> bool:
+        """Delete a downloaded file that `attach_in_place` has finished with,
+        when keeping it would only leave a second copy or a rejected file on
+        disk. Returns True when the file was removed. Both download paths (a
+        subscription walk's import step and the external-link worker) apply
+        this one policy:
+
+        - attached: the attachments store holds the canonical copy.
+        - imported or superseded ARCHIVE: its new images are in the library and
+          the archive itself is in the attachments store.
+        - duplicate skip: the library already holds the content.
+        - soft skip (too small, too transparent, single colour, not a readable
+          image): the import filters rejected it. A file left on disk with no
+          record is picked up by the next walk and imported again; the seen
+          ledger is what stops it being downloaded again.
+        - failed: the archive probe crashed, and a retry would crash again.
+
+        An imported or superseded media file is kept: it IS the record's file.
+        So is any path an ImageRecord points at, whatever the status, because a
+        re-fetch can write identical bytes over a record's own file. A staging
+        file is kept too: it may be a download still being written, and the
+        temp-file sweep removes it once it is old enough to be abandoned."""
+        if is_temp_staging_file(path):
+            return False
+        if result.status in ("imported", "superseded"):
+            redundant = is_archive(path)
+        else:
+            redundant = result.status in ("attached", "skipped", "failed")
+        if not redundant:
+            return False
+        owned = self.session.execute(
+            select(ImageRecord.id).where(
+                ImageRecord.path.in_({str(path), str(path.resolve())})
+            ).limit(1)
+        ).first()
+        if owned is not None:
+            log.info("keeping %s: it is image %s's file", path.name, owned.id)
+            return False
+        try:
+            path.unlink(missing_ok=True)
+        except OSError as exc:
+            log.warning("could not remove %s: %s", path, exc)
+            return False
+        return True
+
     def _upsert_artist(self, name: str) -> Artist:
         slug = slugify(name)
         artist = self.session.execute(
@@ -1733,6 +1808,81 @@ class Importer:
     # is operator-tunable per surface.
     _SINGLE_COLOR_TOLERANCE = 30
 
+    def _screen_still(
+        self, source: Path,
+    ) -> tuple[ImportResult | None, int | None, int | None]:
+        """Read a still image's size and apply the library's content filters.
+
+        Returns `(refusal, width, height)`. `refusal` is a skip result when
+        the file is not a readable image, is below the minimum size, is mostly
+        transparent or is one colour, and None when the file passes.
+
+        A file whose header verifies but whose pixel data is truncated only
+        fails once PIL decodes it, as the transparency check does. That decode
+        error is a skip too: raised, it would retry the import task forever,
+        or abort a download run in the middle of its files.
+        """
+        try:
+            with Image.open(source) as im:
+                im.verify()
+            with Image.open(source) as im:
+                width, height = im.size
+                has_alpha = im.mode in ("RGBA", "LA") or (
+                    im.mode == "P" and "transparency" in im.info
+                )
+        except Exception as exc:
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.invalid_image,
+                error=str(exc),
+            ), None, None
+
+        if (self.settings.min_width and width < self.settings.min_width) or (
+            self.settings.min_height and height < self.settings.min_height
+        ):
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.too_small,
+                error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
+            ), width, height
+
+        if self.settings.skip_transparent and has_alpha:
+            try:
+                pct = self._transparency_pct(source)
+            except OSError as exc:
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.invalid_image,
+                    error=f"PIL load failed during transparency check: {exc}",
+                ), width, height
+            if pct >= self.settings.transparency_threshold:
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.too_transparent,
+                    error=f"{pct:.2%} transparent",
+                ), width, height
+
+        if self.settings.skip_single_color and self._single_color_hit(source):
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.single_color,
+                error=(f"one color dominates >"
+                       f"{self.settings.single_color_threshold:.0%}"),
+            ), width, height
+
+        return None, width, height
+
+    def _phash_of(self, source: Path) -> tuple[ImportResult | None, object]:
+        """The perceptual hash of a still image, as `(refusal, phash)`.
+
+        Hashing decodes every pixel, so this is where a file with truncated
+        pixel data can first fail; that is a skip rather than an exception,
+        for the same reason as in `_screen_still`.
+        """
+        try:
+            with Image.open(source) as im:
+                return None, compute_phash(im)
+        except OSError as exc:
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.invalid_image,
+                error=f"PIL load failed during phash compute: {exc}",
+            ), None
+
     def _single_color_hit(self, source: Path) -> bool:
         """True when one color dominates beyond the configured threshold — the
         same canonical predicate the Cleanup audit runs (audits.single_color,

backend/app/services/ingest_core.py

@@ -36,7 +36,8 @@ from datetime import UTC, datetime, timedelta
 from sqlalchemy import delete, func, select, text
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
-from .gallery_dl import (
+from ..models import ImageRecord
+from .download_result import (
     DownloadResult,
     ErrorType,
     classify_tier_gated,
@@ -105,6 +106,38 @@ _LIVE_PROGRESS_INTERVAL = 5.0
 # recapture (the operator's schema-test flow) reaches the sample.
 _CANARY_MIN_SAMPLE = 30
 
+# The walk's time budget covers only the walk, but phase 3 runs in the SAME
+# Celery task, under the same soft limit (tasks/download.py: 1350s). A walk that
+# finds a lot of work for phase 3 must stop early and leave it to the next chunk,
+# or phase 3 is killed mid-import: TamadaHeijun's recapture, 2026-09-24, walked
+# for ~2 min and then spent 20 min importing 431 orphans and relinking ~3000
+# on-disk files, and died at the soft limit.
+#
+# So the walk also stops when its elapsed time PLUS phase 3's estimated cost
+# would pass CHUNK_TOTAL_SECONDS. An import (hash, pHash, sidecar, provenance,
+# thumbnail enqueue) runs 2-5s depending on the files; the estimate takes the
+# slow end. A relink is a sha256 over NFS plus a lookup.
+#
+# The estimate is a plan, not a guard. PHASE3_DEADLINE_SECONDS is the guard:
+# phase 3 starts no import past it (download_service), and leaves the rest on
+# disk, unmarked, for the next chunk. It sits under the soft limit by enough to
+# finish the import in hand, upsert post records and finalize the event.
+# test_download_source_task pins both under the soft limit.
+CHUNK_TOTAL_SECONDS = 1200.0
+PHASE3_IMPORT_SECONDS = 5.0
+PHASE3_RELINK_SECONDS = 0.25
+PHASE3_DEADLINE_SECONDS = 1200.0
+
+
+def _unskipped(
+    fetched: list[tuple[str, str]],
+    written_keys: dict[str, tuple[str, str]],
+    skip_paths,
+) -> list[tuple[str, str]]:
+    """`fetched` without the ledger keys of files phase 3 left unimported."""
+    skip = {written_keys[p] for p in skip_paths if p in written_keys}
+    return [kv for kv in fetched if kv not in skip] if skip else list(fetched)
+
 
 def _parse_published(raw: object) -> datetime | None:
     """An ISO-8601 post date from either native client, as aware UTC.
@@ -247,6 +280,14 @@ class Ingester:
         # operator-driven "re-read every body" pass; a horizon there would be a
         # third overlapping answer to a question that has two.
         post_meta = getattr(self.client, "post_meta", None)
+        # #4413: optional client seam for a source that is several feeds walked
+        # one after another (a Discord server: every channel and thread). The
+        # tick early-out means "this feed has nothing new", and without the
+        # seam it ends the WHOLE walk — so the first quiet channel would hide
+        # every channel after it. With it, the early-out asks the client to
+        # move on and the walk continues. Absent → the early-out ends the walk,
+        # exactly as before (Patreon and SubscribeStar are one feed each).
+        skip_feed = getattr(self.client, "skip_feed", None)
         horizon: datetime | None = None
         if mode == "tick" and revisit_days > 0 and post_meta is not None:
             horizon = datetime.now(UTC) - timedelta(days=revisit_days)
@@ -261,6 +302,20 @@ class Ingester:
         # source_filehash and (b) link the on-disk image to its Post (#1288) —
         # WITHOUT re-downloading or unlinking the file. Empty outside recapture.
         relink: list[tuple[str, str, str]] = []
+        # Media handed to phase 3 for import. Marked seen by phase 3 once the
+        # import has run (`mark_seen_after_import`), not here — see there.
+        fetched: list[tuple[str, str]] = []
+        # Per entry of `written`: the cursor of the page it came from, and its
+        # ledger key. Phase 3 can stop importing at its deadline; the files it
+        # leaves must stay unmarked, and the next chunk has to resume on the
+        # page holding the first of them, or no walk sees them again.
+        written_cursors: list[str | None] = []
+        written_keys: dict[str, tuple[str, str]] = {}
+        # Post-record keys written this walk. Marked with the media, after
+        # phase 3 has upserted the records — marking them at write time left a
+        # walk killed before phase 3 with posts the ledger calls recorded that
+        # the database never dated, and no later tick walks back to them (#4436).
+        recorded: list[tuple[str, str]] = []
         downloaded = 0
         errors = 0
         quarantined = 0
@@ -292,6 +347,7 @@ class Ingester:
         reached_bottom = False
         budget_hit = False
         early_out = False
+        feeds_caught_up = 0  # #4413: feeds a tick left early via skip_feed
         stopped = False  # plan #708 B4: operator hit Stop mid-walk
         cancel_armed = False  # latched once we observe a live "running" state
 
@@ -313,6 +369,11 @@ class Ingester:
                 written_paths=written,
                 post_record_paths=list(post_records),
                 relink_source_paths=list(relink),
+                written_cursors=list(written_cursors),
+                mark_seen_after_import=lambda skip_paths=(): self._mark_seen(
+                    source_id,
+                    _unskipped(fetched, written_keys, skip_paths) + recorded,
+                ),
                 stdout="\n".join(log_lines),
                 stderr="",
                 return_code=return_code,
@@ -388,7 +449,17 @@ class Ingester:
 
                 # Time-box check at the post boundary (coarse, like a gallery-dl
                 # chunk). Backfill/recovery resume from emitted_cursor next chunk.
-                if time.monotonic() - start >= time_budget_seconds:
+                # The second half is phase 3's share of the task — see
+                # CHUNK_TOTAL_SECONDS. A mid-page stop resumes the same page.
+                elapsed = time.monotonic() - start
+                phase3 = (
+                    len(written) * PHASE3_IMPORT_SECONDS
+                    + len(relink) * PHASE3_RELINK_SECONDS
+                )
+                if (
+                    elapsed >= time_budget_seconds
+                    or elapsed + phase3 >= CHUNK_TOTAL_SECONDS
+                ):
                     budget_hit = True
                     break
 
@@ -471,7 +542,7 @@ class Ingester:
                                     posts_with_body += 1
                             if rec.path is not None:
                                 post_records.append(str(rec.path))
-                                self._mark_seen(source_id, [(pkey, ppid)])
+                                recorded.append((pkey, ppid))
                             # Per-post handling line in the run stdout (the existing
                             # "Raw stdout" panel) — the downloader already read the
                             # post; we only format its outcome here. post_type beside
@@ -512,6 +583,13 @@ class Ingester:
                     recapture=recapture,
                 )
 
+                # An on-disk file is only "done" if something imported it. One
+                # with no ImageRecord at its path was written by a run that died
+                # before phase 3 — it goes to import, not to the ledger.
+                imported_paths = self._recorded_paths([
+                    str(o.path) for o in outcomes
+                    if o.status == "skipped_disk" and o.path is not None
+                ])
                 to_mark: list[tuple[str, str]] = []
                 to_clear: list[str] = []  # recovered → drop any dead-letter row
                 to_fail: list[tuple[str, str, str]] = []  # (key, post_id, error)
@@ -523,11 +601,33 @@ class Ingester:
                         downloaded += 1
                         if outcome.path is not None:
                             written.append(str(outcome.path))
-                        to_mark.append((key, media_item.post_id))
+                            written_cursors.append(emitted_cursor)
+                            written_keys[str(outcome.path)] = (key, media_item.post_id)
+                        fetched.append((key, media_item.post_id))
                         to_clear.append(key)
                         consecutive_seen = 0
+                    elif (
+                        outcome.status == "skipped_disk"
+                        and outcome.path is not None
+                        and str(outcome.path) not in imported_paths
+                    ):
+                        # On disk, never imported: a prior run wrote it and died
+                        # before phase 3. Import it now. Safe to feed to
+                        # attach_in_place because no record owns this path —
+                        # the unlink below is about a file that IS the record.
+                        written.append(str(outcome.path))
+                        written_cursors.append(emitted_cursor)
+                        written_keys[str(outcome.path)] = (key, media_item.post_id)
+                        fetched.append((key, media_item.post_id))
+                        to_clear.append(key)
+                        skipped_count += 1
+                        consecutive_seen += 1
+                        log_lines.append(
+                            f"  post {media_item.post_id} — on disk but never "
+                            f"imported: {outcome.path.name}"
+                        )
                     elif outcome.status == "skipped_disk":
-                        # Already on disk (a prior run). Reconcile the ledger so a
+                        # Already on disk and imported. Reconcile the ledger so a
                         # later tick skips it at tier-1 without a disk stat, but
                         # do NOT re-feed it to phase 3 — attach_in_place would see
                         # the duplicate sha256 and unlink the on-disk copy.
@@ -618,9 +718,15 @@ class Ingester:
                     })
 
                 if early_out:
-                    break
+                    if skip_feed is None:
+                        break
+                    skip_feed()
+                    feeds_caught_up += 1
+                    early_out = False
+                    consecutive_seen = 0
             else:
-                reached_bottom = True
+                # A walk that left feeds early did not read to their ends.
+                reached_bottom = not feeds_caught_up
         except self._error_base as exc:
             # The platform's client-error base — _failure_result (adapter)
             # maps it to a typed error.
@@ -668,6 +774,7 @@ class Ingester:
                 f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
                 if revisited else ""
             )
+            + (f", {feeds_caught_up} feed(s) caught up" if feeds_caught_up else "")
             + (", reached end" if reached_bottom else "")
             + (", time-boxed" if budget_hit else "")
         )
@@ -681,7 +788,13 @@ class Ingester:
             # next chunk resumes from the emitted cursor. No progress → TIMEOUT,
             # which feeds download_service's backfill stall-guard. rc<0 mirrors
             # subprocess TimeoutExpired so completion detection stays false.
-            made_progress = downloaded > 0 or emitted_cursor != resume_cursor
+            # Work handed to phase 3 is progress too: a recapture chunk that
+            # stopped for its imports downloaded nothing, and may not have left
+            # its first page.
+            made_progress = (
+                downloaded > 0 or bool(written) or bool(relink)
+                or emitted_cursor != resume_cursor
+            )
             if made_progress:
                 return _result(
                     success=False, return_code=-1,
@@ -887,6 +1000,16 @@ class Ingester:
             )
             session.commit()
 
+    def _recorded_paths(self, paths: list[str]) -> set[str]:
+        """Which of `paths` an ImageRecord already points at."""
+        if not paths:
+            return set()
+        with self.session_factory() as session:
+            rows = session.execute(
+                select(ImageRecord.path).where(ImageRecord.path.in_(paths))
+            ).scalars().all()
+        return set(rows)
+
     def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
         """Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
 

backend/app/services/library_scan.py

@@ -0,0 +1,181 @@
+"""Files in the artists' folders that the library has no record of.
+
+A download writes its files into the artist's folder and imports them in a
+later step. A run that stops between the two (a restart, a time limit) leaves
+complete files on disk that no image or attachment points at, and a walk that
+has already marked them seen never comes back for them. `find_unimported`
+lists those files; `queue_unimported` turns the same list into an import batch
+that attaches each file where it lies (`scan_mode="library"`), so a file the
+library already holds, or one the import filters reject, is removed instead.
+
+Only files under a folder named for an existing artist (matched by slug, so a
+differently-cased folder still matches) are considered, and only images,
+videos and archives. Files changed in the last day are left for the download
+that may still be writing them.
+"""
+from __future__ import annotations
+
+import os
+import stat
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..models import Artist, ImageRecord, ImportBatch, ImportTask, PostAttachment, Source
+from ..utils.slug import slugify
+from ..utils.staging import is_temp_staging_file
+from .archive_extractor import is_archive
+from .importer import is_supported
+
+# Top-level folders that hold the library's own stores, not an artist's files.
+_STORE_DIRS = frozenset({"thumbs", "attachments", "cookies", "secrets"})
+MIN_AGE_SECONDS = 24 * 3600
+_ACTIVE_TASK_STATUSES = ("pending", "queued", "processing")
+_SAMPLES_PER_ARTIST = 3
+_ARTISTS_LISTED = 50
+
+
+@dataclass(frozen=True)
+class Unimported:
+    path: Path
+    artist_slug: str
+    task_type: str  # "media" or "archive", as ImportTask.task_type
+    size: int
+
+
+def _referenced_paths(session: Session) -> set[str]:
+    paths: set[str] = set()
+    for p, thumb in session.execute(select(ImageRecord.path, ImageRecord.thumbnail_path)):
+        paths.add(p)
+        if thumb:
+            paths.add(thumb)
+    paths.update(session.execute(select(PostAttachment.path)).scalars())
+    paths.update(session.execute(
+        select(ImportTask.source_path).where(ImportTask.status.in_(_ACTIVE_TASK_STATUSES))
+    ).scalars())
+    return paths
+
+
+def find_unimported(
+    session: Session, images_root: Path, *, now: float | None = None,
+) -> tuple[list[Unimported], dict]:
+    """Walk the artists' folders and return (files to import, summary).
+
+    The summary counts what was found and what was passed over, and lists the
+    artists with the most to import (bytes first) with a few sample paths each:
+        {"files", "bytes", "archives", "artists": [{slug, files, bytes,
+         samples}], "recent", "not_media", "no_artist"}"""
+    now = time.time() if now is None else now
+    referenced = _referenced_paths(session)
+    slugs = set(session.execute(select(Artist.slug)).scalars())
+    found: list[Unimported] = []
+    passed = {"recent": 0, "not_media": 0, "no_artist": 0}
+    root = Path(images_root)
+    for top in sorted(os.listdir(root)):
+        if top in _STORE_DIRS or top.startswith(("_", ".")):
+            continue
+        top_dir = root / top
+        if not top_dir.is_dir():
+            continue
+        slug = slugify(top)
+        known = slug in slugs
+        for dirpath, dirnames, filenames in os.walk(top_dir):
+            dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
+            for name in sorted(filenames):
+                path = Path(dirpath) / name
+                if (
+                    name.startswith(".") or path.suffix.lower() == ".json"
+                    or is_temp_staging_file(path)
+                ):
+                    continue
+                if str(path) in referenced:
+                    continue
+                try:
+                    st = path.lstat()
+                except OSError:
+                    continue
+                if not stat.S_ISREG(st.st_mode):
+                    continue
+                kind = "archive" if is_archive(path) else ("media" if is_supported(path) else None)
+                if kind is None:
+                    passed["not_media"] += 1
+                elif now - st.st_mtime < MIN_AGE_SECONDS:
+                    passed["recent"] += 1
+                elif not known:
+                    passed["no_artist"] += 1
+                else:
+                    found.append(Unimported(path, slug, kind, st.st_size))
+
+    per_artist: dict[str, dict] = defaultdict(lambda: {"files": 0, "bytes": 0, "samples": []})
+    for u in found:
+        a = per_artist[u.artist_slug]
+        a["files"] += 1
+        a["bytes"] += u.size
+        if len(a["samples"]) < _SAMPLES_PER_ARTIST:
+            a["samples"].append(str(u.path.relative_to(root)))
+    artists = sorted(
+        ({"slug": s, **v} for s, v in per_artist.items()), key=lambda a: -a["bytes"],
+    )[:_ARTISTS_LISTED]
+    summary = {
+        "files": len(found),
+        "bytes": sum(u.size for u in found),
+        "archives": sum(1 for u in found if u.task_type == "archive"),
+        "artists": artists,
+        **passed,
+    }
+    return found, summary
+
+
+def queue_unimported(
+    session: Session, images_root: Path, *, triggered_by: str = "manual",
+) -> dict:
+    """Create a `library` import batch holding every file `find_unimported`
+    lists, and queue it. Returns the same summary plus `batch_id` (None when
+    there was nothing to import)."""
+    from ..tasks.import_file import enqueue_import  # tasks import services
+
+    found, summary = find_unimported(session, images_root)
+    summary["batch_id"] = None
+    if not found:
+        return summary
+    batch = ImportBatch(
+        triggered_by=triggered_by, source_path=str(images_root),
+        scan_mode="library", status="running", total_files=len(found),
+    )
+    session.add(batch)
+    session.flush()
+    tasks = [
+        ImportTask(
+            batch_id=batch.id, source_path=str(u.path), task_type=u.task_type,
+            status="queued", size_bytes=u.size,
+        )
+        for u in found
+    ]
+    session.add_all(tasks)
+    session.commit()
+    for t in tasks:
+        enqueue_import(t.id, t.task_type)
+    summary["batch_id"] = batch.id
+    return summary
+
+
+def folder_owner(
+    session: Session, path: Path, images_root: Path,
+) -> tuple[Artist | None, Source | None]:
+    """The artist whose folder holds `path`, and that artist's source for the
+    platform folder below it when there is exactly one (a Discord artist has a
+    source per channel, so there the post's sidecar decides)."""
+    parts = Path(path).relative_to(images_root).parts
+    artist = session.execute(
+        select(Artist).where(Artist.slug == slugify(parts[0]))
+    ).scalar_one_or_none()
+    if artist is None or len(parts) < 3:
+        return artist, None
+    sources = session.execute(
+        select(Source).where(Source.artist_id == artist.id, Source.platform == parts[1])
+    ).scalars().all()
+    return artist, (sources[0] if len(sources) == 1 else None)

backend/app/services/ml/heads.py

@@ -97,8 +97,8 @@ def _sigmoid(z, np):
 
 def _conflict_scores(Xn, Wc, bc, np):
     """The presentation conflict signal (#141): per row, the MAX content-head
-    probability and WHICH head produced it. Shared by the system-tag sweep's guard-2
-    and the soft-wip audit — both ask "does this ALSO look like real content?"."""
+    probability and WHICH head produced it — the system-tag sweep's guard-2 asks
+    "does this ALSO look like real content?"."""
     cprobs = _sigmoid(Xn @ Wc.T + bc, np)
     return cprobs.max(axis=1), cprobs.argmax(axis=1)
 
@@ -106,10 +106,9 @@ def _conflict_scores(Xn, Wc, bc, np):
 def _insert_presentation_review(
     session, *, image_record_id, tag_id, conflict_tag_id, conflict_score, mode,
 ):
-    """Single-source the ring-loud PresentationReview row shape so the two writers
-    (system-tag sweep guard-2 + soft-wip audit) can't drift on columns or `mode` —
-    they share the (image_record_id, tag_id) composite PK, so a divergent `mode`
-    would be a silent first-writer-wins bug."""
+    """Single-source the ring-loud PresentationReview row shape, so every writer of
+    the (image_record_id, tag_id) composite PK agrees on columns and `mode` — a
+    divergent `mode` would be a silent first-writer-wins bug."""
     session.execute(
         pg_insert(PresentationReview)
         .values(
@@ -963,68 +962,6 @@ def system_tag_auto_apply_sweep(
     }
 
 
-def soft_wip_conflict_audit(session: Session, dry_run: bool = False) -> dict:
-    """Ring-loud audit for the SOFT WIP-title cohort (#1474). Images auto-tagged
-    `wip` from a low-precision sketch/doodle title (source='wip_title_soft') that ALSO
-    score >= the process conflict threshold on a content head are probably FINISHED
-    art mis-tagged as process — flag them (PresentationReview, mode='process') so the
-    review strip surfaces them ("also looks like <X>", Keep tag / Remove tag). Does
-    NOT remove the tag; the operator decides. No-op when there are no content heads.
-    numpy-only. Returns {n_scanned, n_flagged}."""
-    import numpy as np
-
-    from ..wip_title import WIP_TITLE_SOFT_SOURCE, resolve_wip_tag_id
-
-    settings = _settings(session)
-    ver = settings.embedder_model_version
-    conflict_thr = float(settings.process_conflict_threshold)
-    conf = _conflict_heads(session, ver)
-    wip_id = resolve_wip_tag_id(session)
-    if not conf or wip_id is None:
-        return {"n_scanned": 0, "n_flagged": 0}
-    Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
-    bc = np.asarray([r.bias for r in conf], dtype=np.float32)
-    conf_tag_ids = [r.tag_id for r in conf]
-
-    soft_ids = [iid for (iid,) in session.execute(
-        select(image_tag.c.image_record_id)
-        .where(image_tag.c.tag_id == wip_id)
-        .where(image_tag.c.source == WIP_TITLE_SOFT_SOURCE)
-    )]
-    # Skip images already flagged for this tag (idempotent re-runs).
-    flagged = {iid for (iid,) in session.execute(
-        select(PresentationReview.image_record_id)
-        .where(PresentationReview.tag_id == wip_id)
-    )}
-    soft_ids = [i for i in soft_ids if i not in flagged]
-
-    n_flagged = 0
-    scanned = 0
-    for start in range(0, len(soft_ids), _AUTO_APPLY_CHUNK):
-        chunk = soft_ids[start:start + _AUTO_APPLY_CHUNK]
-        emb = _load_embeddings(session, chunk)
-        cids = [i for i in chunk if i in emb]
-        if not cids:
-            continue
-        scanned += len(cids)
-        Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
-        max_c, arg_c = _conflict_scores(Xn, Wc, bc, np)
-        for k in range(len(cids)):
-            if float(max_c[k]) >= conflict_thr:
-                n_flagged += 1
-                if not dry_run:
-                    _insert_presentation_review(
-                        session,
-                        image_record_id=cids[k], tag_id=wip_id,
-                        conflict_tag_id=conf_tag_ids[int(arg_c[k])],
-                        conflict_score=float(max_c[k]),
-                        mode="process",
-                    )
-        if not dry_run:
-            session.commit()
-    return {"n_scanned": scanned, "n_flagged": n_flagged}
-
-
 def retract_auto_applied_heads(session: Session) -> int:
     """Soft auto-apply (milestone 139): re-score every standing source='head_auto'
     tag against its CURRENT head and REMOVE the ones now BELOW the head's

backend/app/services/ml/training_data.py

@@ -32,11 +32,8 @@ from ...models.tag import image_tag
 # `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
 # ALSO provisional — the head must learn only from title (`wip_title`) + manual
 # labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
-# `wip_title_soft` (#1474): the soft title tier (sketch/doodle) is LOW-precision, so
-# it's provisional too — a finished piece titled "sketch" must not train the wip head.
 _AUTO_SOURCES = (
     "head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
-    "wip_title_soft",
 )
 
 

backend/app/services/native_ingest_common.py

@@ -19,6 +19,7 @@ import json
 import logging
 import os
 import time
+from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
@@ -184,6 +185,56 @@ def post_dir_name(post: dict) -> str:
     return sanitize_segment(raw)
 
 
+# -- one downloadable file, and its seen-ledger identity -------------------
+
+@dataclass
+class MediaItem:
+    """One resolved downloadable file belonging to a post, as every platform
+    client returns it.
+
+    - `url`: what to fetch. `filename`: the media's own name when the platform
+      gives one, else the URL basename (never a network call).
+    - `kind`: the content type, using gallery-dl's `files` names ("images",
+      "attachments", …) so per-source content_types filter the same way.
+    - `post_id`: the owning post, so a flattened list stays traceable.
+    - `filehash`: the 32-hex content hash in a content-addressed CDN URL
+      (Patreon), else None.
+    - `media_id`: the platform's stable per-file id where the URL has no hash:
+      SubscribeStar's gallery id, Discord's attachment id (or, for an embed, a
+      hash of its URL path; the query string is a signature that changes on
+      every fetch). Empty when the platform has none.
+    - `extension` / `num`: Discord only. gallery-dl splits the name from its
+      extension and numbers files 1-based across the message; both name the
+      file on disk.
+    """
+
+    url: str
+    filename: str
+    kind: str
+    post_id: str
+    filehash: str | None = None
+    media_id: str = ""
+    extension: str = ""
+    num: int = 0
+
+
+# The seen-ledger tables' key column width.
+_LEDGER_KEY_MAX = 128
+
+
+def ledger_key(media: MediaItem) -> str:
+    """A file's identity in the cross-run seen-ledger, stable across walks.
+
+    The content hash when the URL carries one. Otherwise the file is keyed
+    within its post: by the platform's per-file id, or by filename when there is
+    none (a Patreon Mux video or a hashless inline image). A positional key is
+    never used, because an edit that removes a file renumbers the ones after it
+    and would make a different file read as seen."""
+    if media.filehash:
+        return media.filehash
+    return f"{media.post_id}:{media.media_id or media.filename}"[:_LEDGER_KEY_MAX]
+
+
 # -- per-item download outcomes (shared dataclasses) -----------------------
 
 @dataclass
@@ -267,11 +318,16 @@ class Membership:
 # -- base downloader (shared fetch/validate plumbing) ----------------------
 
 class BaseNativeDownloader:
-    """Shared download plumbing for native-platform downloaders: the streaming
-    GET (transient-retry + Range-resume) and file validation/quarantine. Platform
-    downloaders subclass this and implement `download_post` / `write_post_record`
-    / the per-media sidecar (and any platform-specific fetch, e.g. Patreon's
-    Mux/yt-dlp video branch). PURE: no DB; the seen-skip is an injected predicate.
+    """Shared download plumbing for native-platform downloaders: one post's
+    files (`download_post`, with the seen/disk skips and the time-box), the
+    streaming GET (transient-retry + Range-resume), and file
+    validation/quarantine.
+
+    A platform downloader subclasses this, implements `write_post_record`, and
+    overrides only the seams where its layout differs: `_post_folder`,
+    `_media_name`, `_media_sidecar`, and for Patreon's Mux/yt-dlp video,
+    `_existing_output` and `_fetch_media`. PURE: no DB; the seen-skip is an
+    injected predicate.
     """
 
     def __init__(
@@ -383,18 +439,118 @@ class BaseNativeDownloader:
         )
         return reason, (dest or path)
 
-    # -- sidecar (per-media, minimal) --------------------------------------
+    # -- one post's files ---------------------------------------------------
 
-    def _write_minimal_sidecar(
-        self, post: dict, media_path: Path, *, source_url: str | None = None
-    ) -> Path:
-        """Post-first per-media sidecar (#856): image identity ONLY
-        (category/id/source_url). The post body/links live solely in _post.json."""
-        data: dict = {"category": self.platform, "id": str(post.get("id") or "")}
-        if source_url:
-            data["source_url"] = source_url
+    def download_post(
+        self,
+        post: dict,
+        media_items: list[MediaItem],
+        artist_slug: str,
+        *,
+        is_seen: Callable[[MediaItem], bool] = lambda m: False,
+        should_stop: Callable[[], bool] = lambda: False,
+        recapture: bool = False,
+    ) -> list[MediaOutcome]:
+        """Download every file of one post and return one outcome per file.
+
+        - Two-tier skip: the injected seen-ledger predicate first, then the disk.
+        - `should_stop()` is polled before each file, so a file-dense post
+          cannot run a time-boxed chunk far past its deadline. The files not
+          reached are not marked seen and are fetched by the next chunk.
+        - One file's failure is an "error" outcome for that file; the rest
+          proceed.
+        - `recapture` re-reads what is on disk without re-downloading: a seen
+          file that is on disk comes back `skipped_disk` with its path (so the
+          engine can backfill its source filehash), and a seen file that is gone
+          is left alone, since refetching is recovery's job.
+        """
+        folder = self._post_folder(post, artist_slug)
+        outcomes: list[MediaOutcome] = []
+        for index, media in enumerate(media_items, start=1):
+            if should_stop():
+                break
+            try:
+                outcomes.append(self._download_one(
+                    post, media, folder, artist_slug, index, is_seen,
+                    recapture=recapture,
+                ))
+            except Exception as exc:  # resilient: isolate one item's failure
+                log.warning(
+                    "%s media failed (post %s, item %d): %s",
+                    self.platform, post.get("id"), index, exc,
+                )
+                outcomes.append(
+                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
+                )
+        return outcomes
+
+    def _download_one(
+        self,
+        post: dict,
+        media: MediaItem,
+        folder: Path,
+        artist_slug: str,
+        index: int,
+        is_seen: Callable[[MediaItem], bool],
+        *,
+        recapture: bool = False,
+    ) -> MediaOutcome:
+        seen = is_seen(media)
+        if seen and not recapture:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+        path = folder / self._media_name(post, media, index)
+        existing = self._existing_output(path, media)
+        if existing is not None:
+            return MediaOutcome(media=media, status="skipped_disk", path=existing, error=None)
+        if seen:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        folder.mkdir(parents=True, exist_ok=True)
+        # Real downloads are paced; the skips above have already returned.
+        if self._rate_limit > 0:
+            time.sleep(self._rate_limit)
+        return self._fetch_media(post, media, path, artist_slug)
+
+    # -- platform seams ----------------------------------------------------
+    # Where a platform lays its files out. Each matches the layout gallery-dl
+    # wrote, so files from before a platform moved to its native downloader
+    # are found on disk instead of fetched again.
+
+    def _post_folder(self, post: dict, artist_slug: str) -> Path:
+        return self.images_root / artist_slug / self.platform / post_dir_name(post)
+
+    def _media_name(self, post: dict, media: MediaItem, index: int) -> str:
+        return sanitize_segment(f"{index:02d}_{media.filename}")
+
+    def _existing_output(self, path: Path, media: MediaItem) -> Path | None:
+        """The file already on disk for this media, or None."""
+        return path if path.exists() else None
+
+    def _fetch_media(
+        self, post: dict, media: MediaItem, path: Path, artist_slug: str,
+    ) -> MediaOutcome:
+        """Fetch one file, validate it, and write its sidecar."""
+        out = self._fetch_get(media.url, path)
+        reason, quarantined = self._validate_path(out, artist_slug, media.url)
+        if reason is not None:
+            # Distinct from an error, so the run reports a real
+            # files_quarantined count and the paths.
+            return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
+        self._write_media_sidecar(post, out, media)
+        return MediaOutcome(media=media, status="downloaded", path=out, error=None)
+
+    def _media_sidecar(self, post: dict, media: MediaItem) -> dict:
+        """The per-file sidecar's content: the file's identity only. The post's
+        body, title and links live in its post record alone, so they are never
+        copied beside every file."""
+        return {"category": self.platform, "id": str(post.get("id") or ""),
+                "source_url": media.url}
+
+    def _write_media_sidecar(self, post: dict, media_path: Path, media: MediaItem) -> Path:
+        """Write the per-file sidecar beside the file, named like it minus the
+        extension, which is what `find_sidecar` pairs first."""
         sidecar_path = media_path.with_suffix(".json")
-        sidecar_path.write_text(json.dumps(data, indent=2))
+        sidecar_path.write_text(json.dumps(self._media_sidecar(post, media), indent=2))
         return sidecar_path
 
 # --- membership status vocabulary (#387) ------------------------------------

backend/app/services/patreon_client.py

@@ -38,14 +38,15 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 
 from __future__ import annotations
 
+import base64
+import json
 import logging
 import re
 import time
 from collections.abc import Iterator
-from dataclasses import dataclass
 from html import unescape
 from pathlib import Path
-from urllib.parse import parse_qs, urlsplit
+from urllib.parse import parse_qs, unquote, urlsplit
 
 import requests
 
@@ -53,6 +54,7 @@ from ..utils.paths import filehash_from_url
 from ..utils.prosemirror import post_body_html
 from .native_ingest_common import (
     _MAX_429_RETRIES,
+    MediaItem,
     Membership,
     NativeAuthError,
     NativeDriftError,
@@ -140,29 +142,25 @@ class PatreonDriftError(PatreonAPIError, NativeDriftError):
     """
 
 
-@dataclass
-class MediaItem:
-    """One resolved downloadable item belonging to a post.
+# Patreon's CDN URLs carry a base64 JSON segment of render options, such as
+# eyJ3Ijo2MjB9 for {"w":620}. A locked preview is rendered blurred and says so
+# with a "b" (blur) option. A post the account can view can still embed media
+# from a post it can't, such as a public teaser showing a paid post's image,
+# and that media arrives blurred even though the post itself isn't gated.
+_RENDER_OPTIONS_RE = re.compile(r"/(eyJ[A-Za-z0-9_\-]+(?:=|%3D)*)(?:/|$)")
 
-    Fields:
-      url       — the CDN/download URL to fetch.
-      filename  — media `file_name` when present; otherwise the URL basename
-                  (NEVER a network call). Bounded/sane extension.
-      kind      — one of: "images", "image_large", "attachments", "postfile",
-                  "content". Mirrors gallery-dl's `files` content-type names so
-                  the later step can honor the same per-source content_types.
-      filehash  — the 32-char hex (MD5) segment from the CDN URL, or None if
-                  the URL carries no such segment. Used for in-post dedup and
-                  the cross-run seen-ledger.
-      post_id   — the owning post's id (so a flattened media list stays
-                  traceable to its post).
-    """
 
-    url: str
-    filename: str
-    kind: str
-    filehash: str | None
-    post_id: str
+def is_blurred_preview(url: str) -> bool:
+    """True when a Patreon CDN URL is a blurred locked preview."""
+    m = _RENDER_OPTIONS_RE.search(urlsplit(url).path)
+    if not m:
+        return False
+    token = unquote(m.group(1))
+    try:
+        options = json.loads(base64.urlsafe_b64decode(token + "=" * (-len(token) % 4)))
+    except (ValueError, TypeError):
+        return False
+    return isinstance(options, dict) and bool(options.get("b"))
 
 
 def _filehash(url: str) -> str | None:
@@ -479,6 +477,11 @@ class PatreonClient:
                     )
                 )
 
+        blurred = [i for i in items if is_blurred_preview(i.url)]
+        if blurred:
+            log.info("Patreon post %s: skipped %d blurred locked preview(s)",
+                     post_id, len(blurred))
+            items = [i for i in items if not is_blurred_preview(i.url)]
         return _dedup_by_filehash(items)
 
     @staticmethod

backend/app/services/patreon_downloader.py

@@ -43,10 +43,10 @@ from .native_ingest_common import (
     _BACKOFF_CAP_SECONDS,
     _MAX_MEDIA_RETRIES,
     BaseNativeDownloader,
+    MediaItem,
     MediaOutcome,
     PostRecordOutcome,
     post_dir_name,
-    sanitize_segment,
 )
 
 log = logging.getLogger(__name__)
@@ -102,139 +102,33 @@ class PatreonDownloader(BaseNativeDownloader):
         # None in unit tests / when enrichment isn't wanted.
         self._content_fetcher = content_fetcher
 
-    # -- public ------------------------------------------------------------
+    # The download loop, layout and per-file sidecar are BaseNativeDownloader's.
+    # Patreon adds one branch: a Mux/HLS video is fetched with yt-dlp, which may
+    # remux it to a container of its own choosing.
 
-    def download_post(
-        self,
-        post: dict,
-        media_items: list,
-        artist_slug: str,
-        *,
-        is_seen: Callable[[object], bool] = lambda m: False,
-        should_stop: Callable[[], bool] = lambda: False,
-        recapture: bool = False,
-    ) -> list[MediaOutcome]:
-        """Download every media item of one post; return per-item outcomes.
+    def _existing_output(self, path: Path, media: MediaItem) -> Path | None:
+        if path.exists():
+            return path
+        if _is_video_url(media.url):
+            # A remux lands at a different extension; honour one already there.
+            return self._existing_video_output(path)
+        return None
 
-        Builds the post directory, iterates media (1-based NN), applies the
-        two-tier skip (injected is_seen, then disk), downloads (plain GET or
-        yt-dlp for video), writes the sidecar for each freshly-downloaded item,
-        validates, and returns outcomes. Resilient: one media's failure yields
-        an "error" outcome for that item; the rest proceed.
-
-        `should_stop()` is polled BEFORE each media item: a media-dense post can
-        otherwise run a backfill chunk far past its time-box (the engine only
-        re-checks the budget between posts), so we honour the deadline mid-post
-        and return the items done so far — the rest re-fetch next chunk (they
-        were never marked seen). Bounds chunk overrun to one media download.
-
-        `recapture` (#830): don't re-download already-present media, but DO surface
-        on-disk media as `skipped_disk` (with its path) even when the seen-ledger
-        would tier-1 skip it — so the engine can backfill source_filehash for
-        inline-image localization. Genuinely-missing seen media is NOT refetched.
-        """
-        post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
-        outcomes: list[MediaOutcome] = []
-
-        for i, media in enumerate(media_items, start=1):
-            if should_stop():
-                break
-            try:
-                outcomes.append(
-                    self._download_one(
-                        post, media, post_dir, artist_slug, i, is_seen,
-                        recapture=recapture,
-                    )
-                )
-            except Exception as exc:  # resilient: isolate one item's failure
-                log.warning(
-                    "Patreon media failed (post %s, item %d): %s",
-                    post.get("id"), i, exc,
-                )
-                outcomes.append(
-                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
-                )
-        return outcomes
-
-    # -- per-item ----------------------------------------------------------
-
-    def _download_one(
-        self,
-        post: dict,
-        media,
-        post_dir: Path,
-        artist_slug: str,
-        index: int,
-        is_seen: Callable[[object], bool],
-        *,
-        recapture: bool = False,
+    def _fetch_media(
+        self, post: dict, media: MediaItem, path: Path, artist_slug: str,
     ) -> MediaOutcome:
-        # tier-1: seen ledger (injected; no DB here). In recapture mode we DON'T
-        # short-circuit here — we fall through to the disk check so an on-disk
-        # seen file is surfaced as skipped_disk (with its path) for source_filehash
-        # backfill; a seen file that's NOT on disk is left alone (not refetched).
-        seen = is_seen(media)
-        if seen and not recapture:
-            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
-
-        nn = f"{index:02d}"
-        final_name = sanitize_segment(f"{nn}_{media.filename}")
-        media_path = post_dir / final_name
-
-        # tier-2: already on disk.
-        if media_path.exists():
+        if not _is_video_url(media.url):
+            return super()._fetch_media(post, media, path, artist_slug)
+        out_path = self._run_ytdlp(media.url, path, _VIDEO_HEADERS)
+        if out_path is None or not Path(out_path).exists():
             return MediaOutcome(
-                media=media, status="skipped_disk", path=media_path, error=None
+                media=media, status="error", path=None, error="yt-dlp produced no output",
             )
-        # Video may land at a different extension; honor a pre-existing remux.
-        if _is_video_url(media.url):
-            existing = self._existing_video_output(media_path)
-            if existing is not None:
-                return MediaOutcome(
-                    media=media, status="skipped_disk", path=existing, error=None
-                )
-
-        # recapture: a seen item that isn't on disk is NOT re-downloaded (that's
-        # recovery's job) — recapture only re-grabs post text + localizes existing
-        # files. Returns skipped_seen so the run-of-seen / counts stay consistent.
-        if seen:
-            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
-
-        post_dir.mkdir(parents=True, exist_ok=True)
-
-        # Pace real downloads only (the skips above already returned). plan #703.
-        if self._rate_limit > 0:
-            time.sleep(self._rate_limit)
-
-        if _is_video_url(media.url):
-            out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS)
-            if out_path is None or not Path(out_path).exists():
-                return MediaOutcome(
-                    media=media,
-                    status="error",
-                    path=None,
-                    error="yt-dlp produced no output",
-                )
-            out_path = Path(out_path)
-        else:
-            out_path = self._fetch_get(media.url, media_path)
-            reason, quarantine_dest = self._validate_path(
-                out_path, artist_slug, media.url
-            )
-            if reason is not None:
-                # Quarantined (corrupt/invalid) — distinct from a download error
-                # so the run can report a real files_quarantined count + paths.
-                return MediaOutcome(
-                    media=media, status="quarantined",
-                    path=quarantine_dest, error=reason,
-                )
-
-        self._write_sidecar(post, out_path, source_url=media.url)
+        out_path = Path(out_path)
+        self._write_media_sidecar(post, out_path, media)
         return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
 
     # -- video (Mux/HLS via yt-dlp) ----------------------------------------
-    # The plain-GET streaming path (_fetch_get / _fetch_to_file) and
-    # _validate_path are inherited from BaseNativeDownloader.
 
     def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None:
         """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`.
@@ -316,43 +210,12 @@ class PatreonDownloader(BaseNativeDownloader):
                 return cand
         return None
 
-    # -- sidecar -----------------------------------------------------------
-
-    def _write_sidecar(
-        self, post: dict, media_path: Path, *, source_url: str | None = None
+    def _write_post_record_file(
+        self, post: dict, sidecar_path: Path, *, detail_fetch: bool = True,
     ) -> Path:
-        """Write the per-media sidecar next to `media_path` — post-first (#856).
-
-        On the native ingester the POST-RECORD (`write_post_record` → `_post.json`)
-        is the sole writer of the post body/links/metadata, captured once per post
-        BEFORE its media in the walk. So the per-media sidecar carries ONLY
-        image-specific identity: `category` (platform) + `id` (external_post_id, to
-        link provenance to the right Post) + this file's `source_url` (its CDN URL,
-        #830 Phase 2 — the importer persists its filehash so the body's inline
-        `<img src>` remaps to the local copy at render time). No body: writing it
-        next to every image duplicated the post body N+1× and risked divergence
-        (milestone #67). The importer skips post fields for these (post_first).
-        """
-        return self._write_sidecar_data(
-            post, media_path.with_suffix(".json"), source_url=source_url,
-            minimal=True,
-        )
-
-    def _write_sidecar_data(
-        self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
-        minimal: bool = False, detail_fetch: bool = True,
-    ) -> Path:
-        """Serialize the post's metadata to `sidecar_path`. The post-only record
-        (`write_post_record`) writes the FULL post (body/title/date/url); the
-        per-media sidecar (`_write_sidecar`, minimal=True) writes only image
-        identity (category/id/source_url) — post-first (#856). `source_url` is set
-        only for the per-media sidecar — a media-less post has no source file."""
-        if minimal:
-            data = {"category": "patreon", "id": str(post.get("id") or "")}
-            if source_url:
-                data["source_url"] = source_url
-            sidecar_path.write_text(json.dumps(data, indent=2))
-            return sidecar_path
+        """Serialize the full post record (body, title, date, url) to
+        `sidecar_path`. The per-file sidecars carry only the file's identity
+        (BaseNativeDownloader._media_sidecar)."""
         attrs = post.get("attributes") or {}
         title = attrs.get("title")
         # Resolve the body HTML from the feed attrs: legacy flat `content`, else
@@ -387,8 +250,6 @@ class PatreonDownloader(BaseNativeDownloader):
             "published_at": published if isinstance(published, str) else None,
             "url": url if isinstance(url, str) else None,
         }
-        if source_url:
-            data["source_url"] = source_url
         sidecar_path.write_text(json.dumps(data, indent=2))
         return sidecar_path
 
@@ -434,10 +295,10 @@ class PatreonDownloader(BaseNativeDownloader):
                 )
         post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
         post_dir.mkdir(parents=True, exist_ok=True)
-        path = self._write_sidecar_data(
+        path = self._write_post_record_file(
             post, post_dir / "_post.json", detail_fetch=not revisit,
         )
-        # _write_sidecar_data has by now memoized any detail-fetched body onto
+        # _write_post_record_file has by now memoized any detail-fetched body onto
         # post["attributes"]["content"], so re-read it for the FINAL char count.
         body = (post.get("attributes") or {}).get("content")
         body_chars = len(body) if isinstance(body, str) else 0

backend/app/services/patreon_ingester.py

@@ -6,9 +6,8 @@ media → tiered skip → download → mark-seen / record-failures / checkpoint-
 now lives in `ingest_core.Ingester` — it's identical for every platform. This
 module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/
 ledger models/constraints/key into the core and supplies the Patreon-specific
-failure mapping. `download_service.download_source` calls `PatreonIngester.run`
-exactly as before; the public surface (this class, `_ledger_key`,
-`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged.
+failure mapping. `download_service.download_source` calls `PatreonIngester.run`;
+the seen-ledger key is the shared `native_ingest_common.ledger_key`.
 
 Three modes (selected by `download_service` from `config_overrides` state):
   - tick      — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out
@@ -36,38 +35,19 @@ from pathlib import Path
 
 from ..models import PatreonFailedMedia, PatreonSeenMedia
 from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
-from .patreon_client import MediaItem, PatreonAPIError, PatreonClient
+from .native_ingest_common import ledger_key
+from .patreon_client import PatreonAPIError, PatreonClient
 from .patreon_downloader import PatreonDownloader
 from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
 
 __all__ = [
     "DEAD_LETTER_THRESHOLD",
     "PatreonIngester",
-    "_ledger_key",
     "verify_patreon_credential",
 ]
 
 log = logging.getLogger(__name__)
 
-# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any
-# synthesized key so a pathologically long file_name can't overflow the column.
-_LEDGER_KEY_MAX = 128
-
-
-def _ledger_key(media: MediaItem) -> str:
-    """Stable per-media identity for the cross-run seen-ledger.
-
-    A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the
-    natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no
-    content hash at discovery) and the odd inline-content `<img>` pointing at a
-    hashless URL. The plan calls the video case the ``video:<post_id>:<media_id>``
-    sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the
-    stable proxy. Bounded to the column width.
-    """
-    if media.filehash:
-        return media.filehash
-    return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX]
-
 
 class PatreonIngester(Ingester):
     """Walk a Patreon campaign's posts, download unseen media, return a
@@ -124,7 +104,7 @@ class PatreonIngester(Ingester):
             failed_model=PatreonFailedMedia,
             seen_constraint="uq_patreon_seen_media_source_id",
             failed_constraint="uq_patreon_failed_media_source_id",
-            ledger_key=_ledger_key,
+            ledger_key=ledger_key,
             platform="patreon",
             error_base=PatreonAPIError,
             # API_DRIFT message phrasing; the base Ingester._failure_result owns

backend/app/services/platform_lock.py

@@ -17,14 +17,16 @@ import logging
 import redis
 
 from ..config import get_config
+from .download_backends import NATIVE_INGESTER_PLATFORMS
 
 log = logging.getLogger(__name__)
 
-# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
-# here: each runs as a self-pacing subprocess and they're lower-volume. The
-# native-ingester platforms are serialized (one paced scrape/API walk at a time).
-# Add a platform here to cap it to a single concurrent walk.
-SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
+# Platforms walked one-at-a-time: every native-ingester platform, one paced
+# scrape/API walk at a time. gallery-dl platforms are not: each runs as a
+# self-pacing subprocess and they are lower-volume. Discord most of all needs
+# it: every source walks on the operator's ONE user token, and parallel walks
+# on a user account are both how its rate limit trips and what gets it flagged.
+SERIALIZED_PLATFORMS = NATIVE_INGESTER_PLATFORMS
 
 _LOCK_PREFIX = "fc:download_lock:"
 
@@ -58,3 +60,22 @@ def platform_lock(platform: str, *, ttl_seconds: int):
     except redis.RedisError as exc:  # pragma: no cover - broker outage
         log.warning("platform_lock unavailable for %s: %s", platform, exc)
         return None
+
+
+def release_all_platform_locks() -> int:
+    """Drop every serialized platform's lock. Returns how many were held.
+
+    Only for the download lane's boot (#4433). A worker that restarts mid-walk
+    is SIGKILLed past its stop grace, so its `finally` never releases the lock,
+    and the TTL keeps every other source on that platform bouncing for up to
+    27 minutes after the new worker is ready. At boot no walk of ours can be
+    running, so a held lock names a dead one. Assumes one download consumer —
+    the only shape FC deploys; a second replica booting would free a live
+    walk's lock (not corrupt it: the walk runs on, a second walk may overlap it).
+    """
+    try:
+        client = _redis()
+        return int(client.delete(*(f"{_LOCK_PREFIX}{p}" for p in SERIALIZED_PLATFORMS)))
+    except redis.RedisError as exc:  # pragma: no cover - broker outage
+        log.warning("could not release platform locks at boot: %s", exc)
+        return 0

backend/app/services/post_feed_service.py

@@ -479,6 +479,10 @@ class PostFeedService:
             # the UI uses this to explain why a post it linked to is not in the
             # stream.
             "absorbed_by_post_id": post.absorbed_by_post_id,
+            # #4481. The Discord channel a message was posted in: both the native
+            # post record and gallery-dl's sidecars carry it as `channel`. None
+            # on every other platform, and on a Discord post that never said.
+            "channel": _discord_channel(post, source),
             "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
             "source": (
                 {"id": source.id, "platform": source.platform}
@@ -488,3 +492,13 @@ class PostFeedService:
             "thumbnails_more": thumbs_entry["more"],
             "attachments": atts_map.get(post.id, []),
         }
+
+
+def _discord_channel(post: Post, source: Source | None) -> str | None:
+    if source is None or source.platform != "discord":
+        return None
+    raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {}
+    channel = raw.get("channel")
+    if not isinstance(channel, str):
+        return None
+    return channel.strip() or None

backend/app/services/post_record_repair.py

@@ -0,0 +1,86 @@
+"""Date the posts whose record reached the disk but never the database (#4436).
+
+A native walk writes each post's record (`_post.json`, or Discord's
+`<day>_<message id>_post.json`) as it goes, and phase 3 upserts those records
+after the walk — which is when the post, and through it its images, get their
+date. Until #4436 the walk marked a record's post key seen at write time, so a
+walk killed before phase 3 (a restart, a stall) left the post undated with the
+ledger saying it was done, and no later tick walked back that far to fix it.
+
+The record files are still on disk. This finds each undated native post's
+record under its artist's folder and upserts it with the post's OWN source —
+never the (artist, platform) lookup, which picks the artist's first source and
+would misfile a Discord channel's post (#4435). Idempotent: a post that is
+already dated is never looked at, and upserting a record twice changes nothing.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from collections import defaultdict
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..models import Artist, Post, Source
+from ..utils.sidecar import parse_sidecar
+from .download_backends import NATIVE_INGESTER_PLATFORMS
+
+log = logging.getLogger(__name__)
+
+
+def _is_record(path: Path) -> bool:
+    return path.name == "_post.json" or path.name.endswith("_post.json")
+
+
+def _record_id(path: Path) -> str | None:
+    try:
+        data = json.loads(path.read_text("utf-8"))
+    except (OSError, ValueError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    return parse_sidecar(data).external_post_id
+
+
+def date_posts_from_records(session: Session, importer, images_root: Path) -> dict:
+    """Upsert the on-disk record of every undated native post. Returns counts."""
+    rows = session.execute(
+        select(Post.external_post_id, Post.source_id, Post.artist_id, Source.platform)
+        .join(Source, Source.id == Post.source_id)
+        .where(
+            Post.post_date.is_(None),
+            Post.synthesized_by.is_(None),
+            Source.platform.in_(NATIVE_INGESTER_PLATFORMS),
+        )
+    ).all()
+    wanted: dict[tuple[int, str], dict[str, int]] = defaultdict(dict)
+    for epid, source_id, artist_id, platform in rows:
+        wanted[(artist_id, platform)][epid] = source_id
+
+    summary = {"undated": len(rows), "dated": 0, "no_record": 0}
+    for (artist_id, platform), by_epid in wanted.items():
+        artist = session.get(Artist, artist_id)
+        root = Path(images_root) / artist.slug / platform if artist else None
+        if root is None or not root.is_dir():
+            summary["no_record"] += len(by_epid)
+            continue
+        found = 0
+        for record in root.rglob("*post.json"):
+            if not _is_record(record):
+                continue
+            epid = _record_id(record)
+            source_id = by_epid.pop(epid, None) if epid else None
+            if source_id is None:
+                continue
+            source = session.get(Source, source_id)
+            if importer.upsert_post_record(record, artist=artist, source=source):
+                found += 1
+            if not by_epid:
+                break
+        summary["dated"] += found
+        summary["no_record"] += len(by_epid)
+    if summary["undated"]:
+        log.info("date_posts_from_records: %s", summary)
+    return summary

backend/app/services/scheduler_service.py

@@ -115,6 +115,32 @@ async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime
     return active
 
 
+def backfill_ready(source: Source) -> bool:
+    """A deep walk the operator started, with budget left and no failure
+    backing it off — due NOW rather than at its next scheduled check.
+
+    A backfill runs one time-boxed chunk per download (plan #693), and nothing
+    queued the next chunk: each waited for the source's regular interval. At
+    the 8-hour default a freshly armed backfill sat untouched until the next
+    check (the operator armed one on 2026-09-25 and saw nothing happen) and a
+    five-chunk walk took most of two days. The tick's in-flight guard keeps
+    one chunk at a time per source and the platform lock one walk per
+    platform, so "due every tick" means "next chunk as soon as the last one
+    ends".
+
+    The failure gate is what keeps a broken source from retrying every
+    minute: any failed chunk raises `consecutive_failures`, which drops the
+    source back onto its backed-off interval. A chunk that fails to progress
+    twice marks the walk stalled (download_service), which ends it here too.
+    """
+    co = source.config_overrides or {}
+    return (
+        co.get("_backfill_state") == "running"
+        and (source.backfill_runs_remaining or 0) > 0
+        and not (source.consecutive_failures or 0)
+    )
+
+
 async def select_due_sources(session: AsyncSession) -> list[Source]:
     """Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
 
@@ -123,6 +149,9 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
     cooldown is the preventive half of the burst-prevention pair (per-source
     consecutive_failures backoff handles the offending source itself).
 
+    A running backfill (`backfill_ready`) is due on every tick, and whether
+    or not its artist is on auto-check — the operator started it by hand.
+
     Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked
     sources go first, then the longest-since-checked, so the most overdue
     sources hit Celery's FIFO download queue first. Anti-starvation: if
@@ -135,7 +164,6 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
         .options(selectinload(Source.artist))
         .join(Artist, Source.artist_id == Artist.id)
         .where(Source.enabled.is_(True))
-        .where(Artist.auto_check.is_(True))
         .order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
     )).scalars().all()
 
@@ -147,6 +175,11 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
     for s in rows:
         if s.platform in cooldowns:
             continue
+        if backfill_ready(s):
+            due.append(s)
+            continue
+        if not s.artist.auto_check:
+            continue
         interval = compute_effective_interval(s, s.artist, settings)
         if s.last_checked_at is None:
             due.append(s)
@@ -163,6 +196,8 @@ def compute_next_check_at(
     """Return the projected datetime of the next check, or None if never checked."""
     if source.last_checked_at is None:
         return None
+    if backfill_ready(source):
+        return datetime.now(UTC)
     interval = compute_effective_interval(source, artist, settings)
     return source.last_checked_at + timedelta(seconds=interval)
 

backend/app/services/source_service.py

@@ -18,7 +18,8 @@ from ..models import (
     Source,
 )
 from .db_helpers import failing_sources_clause
-from .gallery_dl import ErrorType
+from .download_backends import uses_native_ingester
+from .download_result import ErrorType
 from .membership_reconcile import KEPT_KEY, STOPPED_KEY
 from .membership_roster import gated_reasons_for_sources
 from .platforms import known_platform_keys
@@ -67,6 +68,9 @@ class SourceRecord:
     artist_slug: str
     platform: str
     url: str
+    # alembic 0115: the name the platform gives it, where the URL is opaque
+    # (a Discord link is two ids). None until a walk has read it.
+    display_name: str | None
     enabled: bool
     config_overrides: dict | None
     last_checked_at: str | None
@@ -109,6 +113,7 @@ class SourceRecord:
             "artist_slug": self.artist_slug,
             "platform": self.platform,
             "url": self.url,
+            "display_name": self.display_name,
             "enabled": self.enabled,
             "config_overrides": self.config_overrides,
             "last_checked_at": self.last_checked_at,
@@ -125,6 +130,11 @@ class SourceRecord:
             "backfill_posts": self.backfill_posts,
             "tier_gated_count": self.tier_gated_count,
             "gated_reason": self.gated_reason,
+            # Recover / recapture exist only on the native ingester. Sent so the
+            # UI asks the backend's own predicate instead of keeping a copy of
+            # the platform list — the copy said "patreon, subscribestar" for a
+            # day after Discord went native (milestone 428).
+            "native_ingester": uses_native_ingester(self.platform),
         }
 
 
@@ -295,6 +305,7 @@ class SourceService:
             artist_slug=artist.slug,
             platform=source.platform,
             url=source.url,
+            display_name=source.display_name,
             enabled=source.enabled,
             config_overrides=source.config_overrides,
             last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
@@ -551,8 +562,8 @@ class SourceService:
         whole source); the two flags are mutually exclusive, so arming recapture
         clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
         fresh from the top. The flag is cleared on completion (download_service)
-        and on stop. Recapture is Patreon-only (the native ingester's post-record
-        capture); inert elsewhere. The UI gates the action to Patreon sources."""
+        and on stop. Recapture needs the native ingester's post-record capture,
+        so the UI offers it on native sources only (`native_ingester`)."""
         source = (await self.session.execute(
             select(Source).where(Source.id == source_id)
         )).scalar_one_or_none()

backend/app/services/subscribestar_client.py

@@ -32,7 +32,6 @@ import logging
 import re
 import time
 from collections.abc import Iterator
-from dataclasses import dataclass
 from datetime import datetime
 from html import unescape
 from pathlib import Path
@@ -43,6 +42,7 @@ import requests
 from ..utils.paths import filehash_from_url
 from .native_ingest_common import (
     _MAX_429_RETRIES,
+    MediaItem,
     Membership,
     NativeAuthError,
     NativeDriftError,
@@ -163,24 +163,6 @@ class SubscribeStarDriftError(SubscribeStarAPIError, NativeDriftError):
     silently importing nothing."""
 
 
-@dataclass
-class MediaItem:
-    """One resolved downloadable item belonging to a SubscribeStar post.
-
-    Fields mirror patreon_client.MediaItem (so the downloader is structurally the
-    same) plus `media_id` — the stable per-upload gallery id. SubscribeStar's
-    full-res URL is an opaque `/post_uploads?payload=...` (not content-addressed),
-    so `filehash` is usually None and the ledger keys on `<post_id>:<media_id>`.
-    """
-
-    url: str
-    filename: str
-    kind: str
-    filehash: str | None
-    post_id: str
-    media_id: str
-
-
 def _extr(text: str, start: str, end: str) -> str:
     """Substring between the first `start` and the next `end` after it (gallery-
     dl's `text.extr`); '' when either marker is absent."""

backend/app/services/subscribestar_downloader.py

@@ -23,30 +23,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 from __future__ import annotations
 
 import json
-import logging
-import time
-from collections.abc import Callable
 from pathlib import Path
 
 import requests
 
 from .native_ingest_common import (
     BaseNativeDownloader,
-    MediaOutcome,
     PostRecordOutcome,
     post_dir_name,
-    sanitize_segment,
 )
 
-log = logging.getLogger(__name__)
-
 
 class SubscribeStarDownloader(BaseNativeDownloader):
     """Download resolved SubscribeStar media to gallery-dl's on-disk layout.
-    Subclasses BaseNativeDownloader for the shared streaming GET (transient-retry +
-    Range-resume) and validation/quarantine. No video branch (SubscribeStar serves
-    files directly via /post_uploads) and no detail-fetch (the body is already in
-    the feed HTML). PURE: no DB."""
+    The download loop, per-file layout and sidecars are BaseNativeDownloader's
+    defaults, unchanged: there is no video branch (SubscribeStar serves files
+    directly via /post_uploads) and no detail-fetch (the body is already in the
+    feed HTML). This class adds only the post record. PURE: no DB."""
 
     def __init__(
         self,
@@ -62,110 +55,8 @@ class SubscribeStarDownloader(BaseNativeDownloader):
             validate=validate, rate_limit=rate_limit, session=session,
         )
 
-    # -- public ------------------------------------------------------------
-
-    def download_post(
-        self,
-        post: dict,
-        media_items: list,
-        artist_slug: str,
-        *,
-        is_seen: Callable[[object], bool] = lambda m: False,
-        should_stop: Callable[[], bool] = lambda: False,
-        recapture: bool = False,
-    ) -> list[MediaOutcome]:
-        """Download every media item of one post; return per-item outcomes.
-        Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box,
-        recapture surfacing) minus the video branch."""
-        post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
-        outcomes: list[MediaOutcome] = []
-        for i, media in enumerate(media_items, start=1):
-            if should_stop():
-                break
-            try:
-                outcomes.append(
-                    self._download_one(
-                        post, media, post_dir, artist_slug, i, is_seen,
-                        recapture=recapture,
-                    )
-                )
-            except Exception as exc:  # resilient: isolate one item's failure
-                log.warning(
-                    "SubscribeStar media failed (post %s, item %d): %s",
-                    post.get("id"), i, exc,
-                )
-                outcomes.append(
-                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
-                )
-        return outcomes
-
-    # -- per-item ----------------------------------------------------------
-
-    def _download_one(
-        self,
-        post: dict,
-        media,
-        post_dir: Path,
-        artist_slug: str,
-        index: int,
-        is_seen: Callable[[object], bool],
-        *,
-        recapture: bool = False,
-    ) -> MediaOutcome:
-        seen = is_seen(media)
-        if seen and not recapture:
-            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
-
-        nn = f"{index:02d}"
-        media_path = post_dir / sanitize_segment(f"{nn}_{media.filename}")
-
-        if media_path.exists():  # tier-2: already on disk
-            return MediaOutcome(
-                media=media, status="skipped_disk", path=media_path, error=None
-            )
-        # recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
-        if seen:
-            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
-
-        post_dir.mkdir(parents=True, exist_ok=True)
-        if self._rate_limit > 0:
-            time.sleep(self._rate_limit)
-
-        out_path = self._fetch_get(media.url, media_path)
-        reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
-        if reason is not None:
-            return MediaOutcome(
-                media=media, status="quarantined", path=quarantine_dest, error=reason,
-            )
-        self._write_sidecar(post, out_path, source_url=media.url)
-        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
-
-    # The plain-GET streaming path (_fetch_get / _fetch_to_file) and
-    # _validate_path are inherited from BaseNativeDownloader.
-
-    # -- sidecar -----------------------------------------------------------
-
-    def _write_sidecar(
-        self, post: dict, media_path: Path, *, source_url: str | None = None
-    ) -> Path:
-        """Per-media sidecar — post-first (#856): image identity ONLY
-        (category/id/source_url). The post body/links live solely in _post.json."""
-        return self._write_sidecar_data(
-            post, media_path.with_suffix(".json"), source_url=source_url, minimal=True,
-        )
-
-    def _write_sidecar_data(
-        self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
-        minimal: bool = False,
-    ) -> Path:
-        """Serialize the post's metadata. minimal=True → per-media sidecar (image
-        identity only); else the full post record (body/title/date/url)."""
-        if minimal:
-            data = {"category": "subscribestar", "id": str(post.get("id") or "")}
-            if source_url:
-                data["source_url"] = source_url
-            sidecar_path.write_text(json.dumps(data, indent=2))
-            return sidecar_path
+    def _write_post_record_file(self, post: dict, sidecar_path: Path) -> Path:
+        """Serialize the full post record: body, title and date."""
         attrs = post.get("attributes") or {}
         content = attrs.get("content")
         # SubscribeStar synthesizes the post permalink from the id (matches the
@@ -179,8 +70,6 @@ class SubscribeStarDownloader(BaseNativeDownloader):
             "content": content if isinstance(content, str) else "",
             "published_at": attrs.get("published_at"),
         }
-        if source_url:
-            data["source_url"] = source_url
         sidecar_path.write_text(json.dumps(data, indent=2))
         return sidecar_path
 
@@ -214,7 +103,7 @@ class SubscribeStarDownloader(BaseNativeDownloader):
                 )
         post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
         post_dir.mkdir(parents=True, exist_ok=True)
-        path = self._write_sidecar_data(post, post_dir / "_post.json")
+        path = self._write_post_record_file(post, post_dir / "_post.json")
         body = attrs.get("content")
         body_chars = len(body) if isinstance(body, str) else 0
         return PostRecordOutcome(

backend/app/services/subscribestar_ingester.py

@@ -22,30 +22,18 @@ from pathlib import Path
 
 from ..models import SubscribeStarFailedMedia, SubscribeStarSeenMedia
 from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
-from .subscribestar_client import MediaItem, SubscribeStarAPIError, SubscribeStarClient
+from .native_ingest_common import ledger_key
+from .subscribestar_client import SubscribeStarAPIError, SubscribeStarClient
 from .subscribestar_downloader import SubscribeStarDownloader
 
 __all__ = [
     "DEAD_LETTER_THRESHOLD",
     "SubscribeStarIngester",
-    "_ledger_key",
     "verify_subscribestar_credential",
 ]
 
 log = logging.getLogger(__name__)
 
-_LEDGER_KEY_MAX = 128
-
-
-def _ledger_key(media: MediaItem) -> str:
-    """Stable per-media identity for the cross-run seen-ledger. SubscribeStar's
-    full-res URL is an opaque `/post_uploads?payload=...` (no content hash), so
-    `media.filehash` is normally None and the stable proxy is the gallery item id
-    scoped to its post: `<post_id>:<media_id>`. Bounded to the column width."""
-    if media.filehash:
-        return media.filehash
-    return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
-
 
 class SubscribeStarIngester(Ingester):
     """Walk a SubscribeStar creator's posts, download unseen media, return a
@@ -91,7 +79,7 @@ class SubscribeStarIngester(Ingester):
             failed_model=SubscribeStarFailedMedia,
             seen_constraint="uq_subscribestar_seen_media_source_id",
             failed_constraint="uq_subscribestar_failed_media_source_id",
-            ledger_key=_ledger_key,
+            ledger_key=ledger_key,
             platform="subscribestar",
             error_base=SubscribeStarAPIError,
             # API_DRIFT message phrasing; the base Ingester._failure_result owns

backend/app/services/wip_title.py

@@ -27,13 +27,10 @@ from .image_tag_apply import insert_image_tags
 
 # image_tag.source stamped on title-heuristic WIP tags — distinct from the other
 # apply sources so provenance stays legible and a future undo can target only these.
-# HARD tier ("WIP"/"work in progress") is high-precision → trains the wip head.
+# Only the artist's own "WIP"/"work in progress" label counts — high-precision, so it
+# trains the wip head. A sketch/doodle/scribble tier (#1474) was retired in milestone
+# 430: a "sketch" is usually finished art, and its 6k tags flooded the review strip.
 WIP_TITLE_SOURCE = "wip_title"
-# SOFT tier (sketch/doodle/scribble, #1474) is LOWER-precision — a finished "sketch"
-# is often not WIP. This source is PROVISIONAL (in training_data._AUTO_SOURCES) so it
-# NEVER trains the wip head; a soft-tagged image that also looks like real content is
-# surfaced by the ring-loud audit for review.
-WIP_TITLE_SOFT_SOURCE = "wip_title_soft"
 
 # A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
 # (space/underscore/hyphen separated). The letter-boundary lookarounds are what
@@ -45,20 +42,10 @@ _WIP_RE = re.compile(
     re.IGNORECASE,
 )
 
-# Soft tier: sketch / doodle / scribble (+ plurals), letter-boundary anchored so
-# "sketchbook" / "kadoodle" don't trip it. Deliberately conservative — recall is
-# secondary because the soft source doesn't train the head and the ring-loud audit
-# catches false positives.
-_SOFT_WIP_RE = re.compile(
-    r"(?<![A-Za-z])(?:sketch|sketches|doodle|doodles|scribble|scribbles)(?![A-Za-z])",
-    re.IGNORECASE,
-)
-
-# Coarse SQL prefilters for the backfill sweep — narrow the post scan to rows that
+# Coarse SQL prefilter for the backfill sweep — narrows the post scan to rows that
 # COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
-# Each MUST stay a SUPERSET of its regex or the sweep would silently miss posts.
+# It MUST stay a SUPERSET of the regex or the sweep would silently miss posts.
 WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
-SOFT_WIP_TITLE_SQL_PREFILTER = ("%sketch%", "%doodle%", "%scribble%")
 
 # Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
 # ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
@@ -66,19 +53,12 @@ _INSERT_CHUNK = 5000
 
 
 def matches_wip_title(title: str | None) -> bool:
-    """True when a post title explicitly marks it work-in-progress (HARD tier)."""
+    """True when a post title explicitly marks it work-in-progress."""
     if not title:
         return False
     return _WIP_RE.search(title) is not None
 
 
-def matches_soft_wip_title(title: str | None) -> bool:
-    """True when a title carries a SOFT WIP cue (sketch/doodle/scribble, #1474)."""
-    if not title:
-        return False
-    return _SOFT_WIP_RE.search(title) is not None
-
-
 def resolve_wip_tag_id(session: Session) -> int | None:
     """The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
     return session.execute(

backend/app/services/worker_control.py

@@ -211,11 +211,10 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
         state.reserved += len(reserved.get(hostname, []))
         state.consuming.update(q["name"] for q in queues)
 
-        # `pool.max-concurrency` is the number pool_grow/pool_shrink move and
-        # the number the UI shows. Absent on a worker whose stats did not
-        # answer, which leaves pool=None — unknown, not zero.
-        pool = (stats.get(hostname) or {}).get("pool", {}).get("max-concurrency")
-        if isinstance(pool, int):
+        # Absent on a worker whose stats did not answer, which leaves
+        # pool=None — unknown, not zero.
+        pool = pool_size((stats.get(hostname) or {}).get("pool") or {})
+        if pool is not None:
             state.pools[hostname] = pool
 
     for state in out.values():
@@ -223,6 +222,27 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
     return out
 
 
+def pool_size(pool_stats: dict) -> int | None:
+    """How many processes a prefork pool is running NOW, from `inspect stats`.
+
+    The length of `processes`, not `max-concurrency`. Celery reports
+    `max-concurrency` as the pool's `limit`, set once at boot; `pool_grow` and
+    `pool_shrink` hand straight to billiard and never touch it (celery 5.6
+    `concurrency/prefork.py`: `self.grow = P.grow`). So it read 1 forever on a
+    lane booted at 1, and the sizing sweep, computing `target - 1` on every
+    tick, grew a scheduler capped at 2 to six processes while the System tab
+    showed "6 / 1" (2026-09-24) — and could never shrink one, since 1 - 1 is 0.
+
+    `max-concurrency` is the fallback only for a pool that lists no processes
+    (a non-prefork pool), where it is the best number there is.
+    """
+    procs = pool_stats.get("processes")
+    if isinstance(procs, list):
+        return len(procs)
+    limit = pool_stats.get("max-concurrency")
+    return limit if isinstance(limit, int) else None
+
+
 def effective_slots(target: int) -> int:
     """What a pool can actually be set to. Never below one process.
 

backend/app/tasks/admin.py

@@ -74,6 +74,12 @@ def prune_missing_file_records_task(self) -> dict:
     thumbs: list[str] = []
     last_id = 0
     with SessionLocal() as session:
+        # While a restore is fetching the library back, files are missing
+        # because they haven't arrived yet, not because they are gone.
+        from .backup import restore_going
+        if restore_going(session):
+            return {"checked": 0, "missing": 0, "deleted": 0,
+                    "skipped": "a restore is fetching the library's files"}
         while True:
             rows = session.execute(
                 select(ImageRecord.id, ImageRecord.path, ImageRecord.thumbnail_path)
@@ -459,3 +465,29 @@ def reclaim_orphaned_attachments_task(self, dry_run: bool = True) -> dict:
         return cleanup_service.reclaim_orphaned_attachments(
             session, images_root=IMAGES_ROOT, dry_run=dry_run,
         )
+
+
+@celery.task(
+    name="backend.app.tasks.admin.import_unimported_files_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    # One walk of the artists' folders: a stat per file, so minutes on a large
+    # library on network storage. The limits are the backstop for a wedged
+    # filesystem, not the expected exit.
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def import_unimported_files_task(self, dry_run: bool = True) -> dict:
+    """Find files in the artists' folders that the library has no record of.
+    dry_run (the default) returns what was found, per artist, without changing
+    anything; apply queues them as a `library` import batch, which attaches each
+    file where it lies and removes the ones the library already holds or the
+    import filters reject."""
+    from ..services import library_scan
+
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        if dry_run:
+            _found, summary = library_scan.find_unimported(session, IMAGES_ROOT)
+            return {**summary, "batch_id": None}
+        return library_scan.queue_unimported(session, IMAGES_ROOT)

backend/app/tasks/backup.py

@@ -8,17 +8,27 @@ BackupRun.
 from __future__ import annotations
 
 import logging
-from datetime import UTC, datetime
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 from celery.exceptions import SoftTimeLimitExceeded
-from sqlalchemy import select
+from sqlalchemy import select, text, update
 from sqlalchemy.exc import DBAPIError, OperationalError
 
 from ..celery_app import celery
 from ..config import get_config
 from ..models import BackupRun, ImportSettings
-from ..services import backup_service
+from ..services import (
+    backup_destination,
+    backup_media,
+    backup_restore,
+    backup_s3,
+    backup_service,
+    backup_verify,
+)
+from ._sync_engine import get_sync_engine
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
 log = logging.getLogger(__name__)
@@ -49,11 +59,19 @@ def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
 )
 def backup_db_task(self, *, tag: str | None = None,
                    triggered_by: str = "manual") -> dict:
-    """Create one DB backup. Returns {'backup_run_id': N}."""
+    """Create one DB backup. Returns {'backup_run_id': N}.
+
+    With an off-site destination set up, the dump streams to that bucket and
+    nothing is written here; the row's manifest says where it went
+    (`offsite`: endpoint, bucket, key). Without one, the dump is written to
+    /images/_backups."""
     SessionLocal = _sync_session_factory()
     cfg = get_config()
     now = datetime.now(UTC)
     with SessionLocal() as session:
+        # A restore replaces the database this would dump.
+        if restore_going(session):
+            return {"skipped": "a restore is going"}
         row = BackupRun(
             kind="db", status="running", tag=tag,
             triggered_by=triggered_by, started_at=now, manifest={},
@@ -64,10 +82,17 @@ def backup_db_task(self, *, tag: str | None = None,
         run_id = row.id
 
     try:
-        result = backup_service.backup_db(
-            db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
-            tag=tag, triggered_by=triggered_by,
-        )
+        with SessionLocal() as session:
+            settings = ImportSettings.load_sync(session)
+            target = backup_destination.saved_target(settings)
+            limit_mbps = settings.backup_upload_limit_mbps
+        if target is not None:
+            result = _dump_offsite(cfg, target, limit_mbps)
+        else:
+            result = backup_service.backup_db(
+                db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
+                tag=tag, triggered_by=triggered_by,
+            )
     except (SoftTimeLimitExceeded, Exception) as exc:
         with SessionLocal() as session:
             row = session.get(BackupRun, run_id)
@@ -79,61 +104,23 @@ def backup_db_task(self, *, tag: str | None = None,
         row = session.get(BackupRun, run_id)
         row.status = "ok"
         row.finished_at = datetime.now(UTC)
-        row.sql_path = result["sql_path"]
         row.size_bytes = result["size_bytes"]
-        row.manifest = {
-            "manifest_path": result["manifest_path"],
-            "ts": result["ts"],
-        }
-        session.commit()
-    return {"backup_run_id": run_id}
-
-
-@celery.task(
-    name="backend.app.tasks.backup.backup_images_task",
-    bind=True,
-    autoretry_for=(OperationalError, DBAPIError),
-    retry_backoff=30, retry_backoff_max=300, max_retries=1,
-    soft_time_limit=21600, time_limit=23400,
-)
-def backup_images_task(self, *, tag: str | None = None,
-                       triggered_by: str = "manual") -> dict:
-    """Create one images backup. Same shape as backup_db_task; uses
-    tar_path instead of sql_path."""
-    SessionLocal = _sync_session_factory()
-    now = datetime.now(UTC)
-    with SessionLocal() as session:
-        row = BackupRun(
-            kind="images", status="running", tag=tag,
-            triggered_by=triggered_by, started_at=now, manifest={},
-        )
-        session.add(row)
-        session.commit()
-        session.refresh(row)
-        run_id = row.id
-
-    try:
-        result = backup_service.backup_images(
-            images_root=IMAGES_ROOT,
-            tag=tag, triggered_by=triggered_by,
-        )
-    except (SoftTimeLimitExceeded, Exception) as exc:
-        with SessionLocal() as session:
-            row = session.get(BackupRun, run_id)
-            if row is not None:
-                _mark_failed(session, row, exc)
-        raise
-
-    with SessionLocal() as session:
-        row = session.get(BackupRun, run_id)
-        row.status = "ok"
-        row.finished_at = datetime.now(UTC)
-        row.tar_path = result["tar_path"]
-        row.size_bytes = result["size_bytes"]
-        row.manifest = {
-            "manifest_path": result["manifest_path"],
-            "ts": result["ts"],
-        }
+        if target is not None:
+            row.manifest = {
+                "ts": result["ts"],
+                "offsite": {
+                    "endpoint": result["endpoint"],
+                    "bucket": result["bucket"],
+                    "key": result["key"],
+                    "parts": result["parts"],
+                },
+            }
+        else:
+            row.sql_path = result["sql_path"]
+            row.manifest = {
+                "manifest_path": result["manifest_path"],
+                "ts": result["ts"],
+            }
         session.commit()
     return {"backup_run_id": run_id}
 
@@ -189,52 +176,6 @@ def restore_db_task(self, *, source_backup_run_id: int) -> dict:
     return {"backup_run_id": marker_id}
 
 
-@celery.task(
-    name="backend.app.tasks.backup.restore_images_task",
-    bind=True, max_retries=0,
-    soft_time_limit=21600, time_limit=23400,
-)
-def restore_images_task(self, *, source_backup_run_id: int) -> dict:
-    """Mirrors restore_db_task; uses backup_service.restore_images."""
-    SessionLocal = _sync_session_factory()
-    now = datetime.now(UTC)
-    with SessionLocal() as session:
-        src = session.get(BackupRun, source_backup_run_id)
-        if src is None or src.kind != "images" or not src.tar_path:
-            raise ValueError(
-                f"BackupRun id={source_backup_run_id} is not a valid images backup"
-            )
-        marker = BackupRun(
-            kind="images", status="restoring",
-            triggered_by="restore", started_at=now,
-            restored_from_id=src.id,
-            manifest={"source_tar_path": src.tar_path},
-        )
-        session.add(marker)
-        session.commit()
-        session.refresh(marker)
-        marker_id = marker.id
-        tar_path = src.tar_path
-
-    try:
-        backup_service.restore_images(
-            images_root=IMAGES_ROOT, tar_path=Path(tar_path),
-        )
-    except (SoftTimeLimitExceeded, Exception) as exc:
-        with SessionLocal() as session:
-            row = session.get(BackupRun, marker_id)
-            if row is not None:
-                _mark_failed(session, row, exc)
-        raise
-
-    with SessionLocal() as session:
-        row = session.get(BackupRun, marker_id)
-        row.status = "restored"
-        row.finished_at = datetime.now(UTC)
-        session.commit()
-    return {"backup_run_id": marker_id}
-
-
 @celery.task(
     name="backend.app.tasks.backup.prune_backups",
     soft_time_limit=300, time_limit=600,
@@ -242,34 +183,39 @@ def restore_images_task(self, *, source_backup_run_id: int) -> dict:
 def prune_backups() -> dict:
     """Daily Beat. Per-kind retention from ImportSettings.
 
-    Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
+    Returns {"db_deleted": N, "media_deleted": L, "restore_deleted": R, "reconcile_deleted": C, "test_deleted": T,
+    "files_unlinked": K}. Media, restore, reconcile and test runs leave no
+    files here; only their history is trimmed, to the latest MEDIA_RUNS_KEPT,
+    RESTORE_RUNS_KEPT, RECONCILE_RUNS_KEPT and TEST_RUNS_KEPT.
     Tagged rows (tag IS NOT NULL) are never pruned.
     Status='running' / 'restoring' rows are never pruned — the
     recover_stalled_backup_runs sweep flips truly-stuck ones to
-    'error' first. (Earlier docstring claimed the FC-3i TaskRun sweep
-    handled those, but TaskRun cleanup never touched BackupRun rows.
-    Audit 2026-06-02 added the dedicated sweep.)
+    'error' first.
     """
     SessionLocal = _sync_session_factory()
-    counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
+    counts = {"db_deleted": 0, "media_deleted": 0,
+              "restore_deleted": 0, "reconcile_deleted": 0, "test_deleted": 0,
+              "files_unlinked": 0}
     with SessionLocal() as session:
         s = ImportSettings.load_sync(session)
         for kind, keep in (
             ("db", s.backup_db_keep_last_n),
-            ("images", s.backup_images_keep_last_n),
+            ("media", MEDIA_RUNS_KEPT),
+            ("restore", RESTORE_RUNS_KEPT),
+            ("reconcile", RECONCILE_RUNS_KEPT),
+            ("test", TEST_RUNS_KEPT),
         ):
             candidates = session.execute(
                 select(BackupRun)
                 .where(BackupRun.kind == kind)
                 .where(BackupRun.tag.is_(None))
-                .where(BackupRun.status.in_(["ok", "error"]))
+                .where(BackupRun.status.in_(["ok", "error", "restored"]))
                 .order_by(BackupRun.started_at.desc())
                 .offset(keep)
             ).scalars().all()
             for row in candidates:
                 result = backup_service.unlink_artifact_files(
                     sql_path=row.sql_path,
-                    tar_path=row.tar_path,
                     manifest_path=(row.manifest or {}).get("manifest_path"),
                 )
                 counts["files_unlinked"] += sum(
@@ -300,4 +246,729 @@ def backup_db_nightly() -> dict:
     if now_hour != configured_hour:
         return {"skipped": f"hour={now_hour} != configured={configured_hour}"}
     res = backup_db_task.delay(triggered_by="nightly")
-    return {"dispatched": res.id}
+    out = {"dispatched": res.id}
+    # With an off-site destination, the night's backup is the whole library:
+    # the database dump, then any images and attachments not yet in the bucket.
+    # Both run on the long-maintenance lane, one after the other.
+    with SessionLocal() as session:
+        try:
+            offsite = backup_destination.saved_target(ImportSettings.load_sync(session))
+        except Exception:  # noqa: BLE001 — the media run reports it itself
+            offsite = True
+    if offsite is not None:
+        out["media_dispatched"] = backup_media_task.delay(triggered_by="nightly").id
+    return out
+
+
+# ── Off-site copy of the library's files ─────────────────────────────────────
+
+# Each task invocation sends files for this long, then queues the next chunk
+# of the same run, so other long-maintenance jobs get their turn in between.
+MEDIA_CHUNK_SECONDS = 45 * 60
+MEDIA_RUNS_KEPT = 60
+# A failed file is listed on the run, up to this many; the count covers all.
+MEDIA_FAILURES_LISTED = 20
+# The weekly bucket check keeps about a quarter's history.
+RECONCILE_RUNS_KEPT = 12
+
+
+def _dump_offsite(cfg, target, limit_mbps) -> dict:
+    """Dump the database to the bucket, with its row counts beside it for a
+    test restore to check against. The dump stands without its counts, so a
+    failure to store them is logged, not raised."""
+    result = backup_service.backup_db_offsite(
+        db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps)
+    if result.get("counts"):
+        try:
+            backup_verify.store_counts(target, result["key"], result["counts"])
+        except (backup_s3.S3Error, OSError):
+            log.warning("backup: the row counts for %s weren't stored", result["key"],
+                        exc_info=True)
+    return result
+
+
+def _offsite_manifest(target) -> dict:
+    return {"endpoint": target.endpoint, "bucket": target.bucket,
+            "key": backup_media.PREFIX}
+
+
+# ── One chunk at a time ──────────────────────────────────────────────────────
+#
+# A media backup and a restore's file phase run as a chain of chunks, each
+# queuing the next. The chunk carrying a run writes its token to the run's
+# manifest as "chunk", and hands that token to the chunk it queues as `after`.
+# A chunk queued to resume a run after a restart carries `resume` instead and
+# takes the run over. Any other message is stale (a killed chunk's own message
+# delivered again, say) and is dropped, so a run never goes on twice and the
+# chain never doubles.
+
+
+def _chunk_token(task) -> str:
+    return task.request.id or uuid.uuid4().hex
+
+
+def _claim_run(session, run_id: int, status: str, token: str, *, after: str | None,
+               resume: bool) -> str | None:
+    """Take the run for this chunk. Returns None when taken, or why not."""
+    row = session.get(BackupRun, run_id, with_for_update=True)
+    if row is None or row.status != status:
+        session.rollback()
+        return "the run is no longer going"
+    held = (row.manifest or {}).get("chunk")
+    if not resume and held is not None and held != after:
+        session.rollback()
+        return "another chunk carries this run on"
+    row.manifest = {**(row.manifest or {}), "chunk": token}
+    session.commit()
+    return None
+
+
+def _still_ours(session, run_id: int, token: str) -> bool:
+    """Lock the run's row and say whether this chunk still carries it. A chunk
+    that was taken over stops at its next file rather than write over the
+    newer chunk's place; on False the transaction is already rolled back."""
+    held = session.execute(
+        select(BackupRun.manifest).where(BackupRun.id == run_id).with_for_update()
+    ).scalar_one_or_none()
+    if (held or {}).get("chunk") == token:
+        return True
+    session.rollback()
+    return False
+
+
+def resume_offsite_runs() -> dict:
+    """Queue the next chunk of every off-site run a restart cut short: a media
+    backup still running, and a restore fetching its files. Called when the
+    long-maintenance lane starts, where those chunks run, so none of theirs is
+    alive then; the queued chunk carries on from the run's saved place. A
+    restore stopped while loading its database isn't resumed: the stall sweep
+    ends it, and the screen offers the restore again."""
+    with _sync_session_factory()() as session:
+        media = list(session.execute(
+            select(BackupRun.id).where(BackupRun.kind == "media")
+            .where(BackupRun.status == "running")
+        ).scalars())
+        restores = [
+            r.id for r in session.execute(
+                select(BackupRun).where(BackupRun.kind == "restore")
+                .where(BackupRun.status == "restoring")
+            ).scalars()
+            if (r.manifest or {}).get("phase") == "files"
+        ]
+    try:
+        tests = end_cut_off_tests()
+    except Exception:  # noqa: BLE001 — resuming the runs below matters more
+        log.exception("couldn't end a test restore cut off by the restart")
+        tests = 0
+    for run_id in media:
+        backup_media_task.delay(run_id=run_id, resume=True)
+    for run_id in restores:
+        restore_offsite_task.delay(run_id=run_id, resume=True)
+    if media or restores:
+        log.info("resumed off-site runs after a restart: media %s, restore %s",
+                 media, restores)
+    return {"media": media, "restore": restores, "tests_ended": tests}
+
+
+@celery.task(
+    name="backend.app.tasks.backup.backup_media_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=30, retry_backoff_max=300, max_retries=2,
+    # A chunk stops starting files after MEDIA_CHUNK_SECONDS; the limits leave
+    # room for the file in flight, a multi-gigabyte archive at a low upload
+    # limit included.
+    soft_time_limit=21600, time_limit=23400,
+)
+def backup_media_task(self, *, run_id: int | None = None, triggered_by: str = "manual",
+                      after: str | None = None, resume: bool = False) -> dict:
+    """Send the library's images and attachments that aren't in the off-site
+    bucket yet, one chunk of a run. Called without `run_id` it starts a run
+    (unless one is already going); each chunk queues the next until nothing is
+    left, and a restart resumes the run where it stopped. A file that can't be
+    sent is listed on the run and tried again by the next run. Returns
+    {'backup_run_id': N, ...}."""
+    SessionLocal = _sync_session_factory()
+    now = datetime.now(UTC)
+    token = _chunk_token(self)
+    with SessionLocal() as session:
+        if run_id is None:
+            going = session.execute(
+                select(BackupRun.id).where(BackupRun.kind == "media")
+                .where(BackupRun.status == "running").limit(1)
+            ).scalar_one_or_none()
+            if going is not None:
+                return {"backup_run_id": going, "skipped": "a media backup is already running"}
+            # During a restore the library's files are still arriving.
+            if restore_going(session):
+                return {"skipped": "a restore is going"}
+            row = BackupRun(kind="media", status="running", triggered_by=triggered_by,
+                            started_at=now, last_progress_at=now, size_bytes=0,
+                            manifest={"sent": 0, "failed": 0, "failures": [], "cursor": "",
+                                      "chunk": token})
+            session.add(row)
+            session.commit()
+            run_id = row.id
+        else:
+            why = _claim_run(session, run_id, "running", token, after=after, resume=resume)
+            if why:
+                return {"backup_run_id": run_id, "skipped": why}
+
+    try:
+        with SessionLocal() as session:
+            settings = ImportSettings.load_sync(session)
+            target = backup_destination.saved_target(settings)
+            limit_mbps = settings.backup_upload_limit_mbps
+        if target is None:
+            raise backup_s3.S3Error("No off-site destination is set up.")
+        more = _send_media_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps),
+                                 token)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            if row is not None:
+                _mark_failed(session, row, exc)
+        raise
+
+    if more is None:
+        return {"backup_run_id": run_id, "skipped": "another chunk took the run over"}
+    if more:
+        backup_media_task.delay(run_id=run_id, after=token)
+        return {"backup_run_id": run_id, "continues": True}
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        row.status = "ok"
+        row.finished_at = datetime.now(UTC)
+        if m.get("failed"):
+            first = (m.get("failures") or [{}])[0]
+            row.error = (f"{m['failed']} file(s) weren't sent; the next run tries them again. "
+                         f"First: {first.get('path')}: {first.get('error')}")[:2000]
+        session.commit()
+    return {"backup_run_id": run_id, "continues": False}
+
+
+def _send_media_chunk(SessionLocal, run_id: int, target, pacer, token: str) -> bool | None:
+    """Send files until the chunk's time is up or none are left. Progress is
+    committed after every file, so a chunk that dies loses at most the file
+    in flight. Returns True when files remain for another chunk, and None when
+    another chunk has taken the run over."""
+    tid = backup_media.target_id(target)
+    deadline = time.monotonic() + MEDIA_CHUNK_SECONDS
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        m["offsite"] = _offsite_manifest(target)
+        cursor = m.get("cursor") or ""
+        while True:
+            batch = backup_media.pending(session, tid, after=cursor)
+            # End the read before any upload starts: a transaction left open
+            # across a long transfer holds back vacuum for its whole length.
+            session.commit()
+            if not batch:
+                if not _still_ours(session, run_id, token):
+                    return None
+                m["cursor"] = cursor
+                row.manifest = dict(m)
+                row.last_progress_at = datetime.now(UTC)
+                session.commit()
+                return False
+            for blob in batch:
+                if time.monotonic() > deadline:
+                    if not _still_ours(session, run_id, token):
+                        return None
+                    m["cursor"] = cursor
+                    row.manifest = dict(m)
+                    session.commit()
+                    return True
+                key = None
+                try:
+                    key = backup_media.send(target, blob, IMAGES_ROOT, pacer)
+                except (backup_s3.S3Error, OSError) as exc:
+                    m["failed"] = m.get("failed", 0) + 1
+                    if len(m.get("failures") or []) < MEDIA_FAILURES_LISTED:
+                        m["failures"] = [*(m.get("failures") or []),
+                                         {"path": blob.path, "error": str(exc)[:300]}]
+                if not _still_ours(session, run_id, token):
+                    return None
+                if key is not None:
+                    backup_media.record(session, tid, [(blob, key)])
+                    m["sent"] = m.get("sent", 0) + 1
+                    row.size_bytes = (row.size_bytes or 0) + blob.size_bytes
+                cursor = blob.sha256
+                m["cursor"] = cursor
+                row.manifest = dict(m)
+                row.last_progress_at = datetime.now(UTC)
+                session.commit()
+
+
+@celery.task(
+    name="backend.app.tasks.backup.reconcile_media_backup",
+    soft_time_limit=1800, time_limit=2100,
+)
+def reconcile_media_backup() -> dict:
+    """Weekly Beat. Compare what the off-site bucket holds under `media/` with
+    what FabledCurator believes it sent: a missing object is sent again by
+    the next run, and an object already there is not sent twice. Skipped
+    while a media run is going, and when no destination is set up.
+
+    Each check is kept as a run of kind "reconcile" with its counts
+    ({listed, dropped, adopted}), so the Backups card lists it and the System
+    view can warn when files were missing from the bucket."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        target = backup_destination.saved_target(ImportSettings.load_sync(session))
+        if target is None:
+            return {"skipped": "no destination"}
+        going = session.execute(
+            select(BackupRun.id).where(BackupRun.kind == "media")
+            .where(BackupRun.status == "running").limit(1)
+        ).scalar_one_or_none()
+        if going is not None:
+            return {"skipped": "a media backup is running"}
+        now = datetime.now(UTC)
+        row = BackupRun(kind="reconcile", status="running", triggered_by="weekly",
+                        started_at=now, last_progress_at=now,
+                        manifest={"offsite": _offsite_manifest(target)})
+        session.add(row)
+        session.commit()
+        run_id = row.id
+        try:
+            counts = backup_media.reconcile(session, target)
+        except (SoftTimeLimitExceeded, Exception) as exc:
+            session.rollback()
+            row = session.get(BackupRun, run_id)
+            if row is not None:
+                _mark_failed(session, row, exc)
+            raise
+        row = session.get(BackupRun, run_id)
+        row.status = "ok"
+        row.finished_at = datetime.now(UTC)
+        row.manifest = {**(row.manifest or {}), **counts}
+        session.commit()
+        return {"backup_run_id": run_id, **counts}
+
+
+# ── Restoring from the off-site bucket ───────────────────────────────────────
+
+# The file phase fetches for this long per task, then queues its next chunk,
+# like a media run.
+RESTORE_CHUNK_SECONDS = 45 * 60
+RESTORE_RUNS_KEPT = 20
+RESTORE_FAILURES_LISTED = 20
+# A run over files already on disk notes its place this often, not per file.
+_RESTORE_NOTE_SECONDS = 5
+_RESTORE_COUNTS = ("restored", "copied", "present", "missing", "failed")
+
+
+def backup_busy(session) -> str | None:
+    """Why a backup or a restore can't start right now, or None."""
+    row = session.execute(
+        select(BackupRun.kind).where(BackupRun.status.in_(["running", "restoring"])).limit(1)
+    ).first()
+    if row is None:
+        return None
+    if row.kind == "restore":
+        return "a restore is going"
+    if row.kind == "test":
+        return "a test restore is running"
+    return f"a {row.kind} backup is running"
+
+
+def restore_going(session) -> bool:
+    return session.execute(
+        select(BackupRun.id).where(BackupRun.kind == "restore")
+        .where(BackupRun.status == "restoring").limit(1)
+    ).first() is not None
+
+
+# ── Test restore ─────────────────────────────────────────────────────────────
+
+TEST_RUNS_KEPT = 20
+
+
+@celery.task(
+    name="backend.app.tasks.backup.restore_test_task",
+    bind=True,
+    max_retries=0,
+    # Downloading a dump, loading it and rebuilding its indexes; a restore of
+    # the database alone, with room to spare.
+    soft_time_limit=4 * 3600, time_limit=4 * 3600 + 900,
+)
+def restore_test_task(self, *, dump_key: str | None = None,
+                      source_backup_run_id: int | None = None) -> dict:
+    """Test a database dump without touching this install: load it into a
+    temporary database, check every table against the counts recorded with
+    the dump, bring it up to this version, and drop it again
+    (services/backup_verify.py). Tests the bucket's dump at `dump_key` (the
+    newest when None), or, given `source_backup_run_id`, a dump on this
+    server. The run ends "ok" when every table matched, "error" with what
+    didn't otherwise. Returns {'backup_run_id': N, ...}."""
+    SessionLocal = _sync_session_factory()
+    cfg = get_config()
+    now = datetime.now(UTC)
+    with SessionLocal() as session:
+        if why := restore_test_blocked(session):
+            return {"skipped": why}
+        row = BackupRun(kind="test", status="running", triggered_by="manual",
+                        started_at=now, last_progress_at=now, manifest={"phase": "fetching"})
+        session.add(row)
+        session.commit()
+        run_id = row.id
+
+    def note(phase: str, **more) -> None:
+        with SessionLocal() as session:
+            r = session.get(BackupRun, run_id)
+            _note(session, r, {**r.manifest, "phase": phase, **more})
+
+    staged = offsite = None
+    try:
+        if source_backup_run_id is not None:
+            dump_path, recorded, source = _local_dump(SessionLocal, source_backup_run_id)
+        else:
+            with SessionLocal() as session:
+                settings = ImportSettings.load_sync(session)
+                target = backup_destination.saved_target(settings)
+                limit_mbps = settings.backup_upload_limit_mbps
+            if target is None:
+                raise backup_s3.S3Error("No off-site destination is set up.")
+            dump = backup_restore.find_dump(target, dump_key)
+            source = {"key": dump.key, "ts": dump.ts, "size_bytes": dump.size_bytes}
+            offsite = {**_offsite_manifest(target), "key": dump.key}
+            note("fetching", dump=source, offsite=offsite)
+            recorded = backup_verify.fetch_counts(target, dump.key)
+            staged = backup_restore.fetch_dump(
+                target, dump, IMAGES_ROOT / "_backups", backup_s3.Pacer(limit_mbps))
+            dump_path = staged
+        note("loading", dump=source)
+        result = backup_verify.trial_restore(
+            db_url=cfg.database_url_sync, dump_path=dump_path, recorded=recorded,
+            note=note)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        with SessionLocal() as session:
+            r = session.get(BackupRun, run_id)
+            if r is not None:
+                _mark_failed(session, r, exc)
+        raise
+    finally:
+        if staged is not None:
+            staged.unlink(missing_ok=True)
+
+    with SessionLocal() as session:
+        r = session.get(BackupRun, run_id)
+        r.status = "error" if result["mismatched"] else "ok"
+        r.error = backup_verify.summary(result) if result["mismatched"] else None
+        r.finished_at = datetime.now(UTC)
+        r.size_bytes = source.get("size_bytes")
+        r.manifest = {"phase": "done", "dump": source, "offsite": offsite,
+                      "summary": backup_verify.summary(result), **result}
+        session.commit()
+    return {"backup_run_id": run_id, "mismatched": result["mismatched"]}
+
+
+def _local_dump(SessionLocal, run_id: int) -> tuple[Path, dict | None, dict]:
+    """A dump on this server: its path, its recorded counts, and what the
+    test run says it tested."""
+    with SessionLocal() as session:
+        src = session.get(BackupRun, run_id)
+        if src is None or src.kind != "db" or src.status != "ok" or not src.sql_path:
+            raise backup_s3.S3Error(f"Backup {run_id} isn't a database dump on this server.")
+        path = Path(src.sql_path)
+        if not path.is_file():
+            raise backup_s3.S3Error(f"{path} is no longer on this server.")
+        recorded = backup_verify.local_counts((src.manifest or {}).get("manifest_path"))
+        ts = (src.manifest or {}).get("ts")
+        return path, recorded, {"run_id": run_id, "ts": ts, "size_bytes": src.size_bytes}
+
+
+def restore_test_blocked(session) -> str | None:
+    """Why a test restore can't start right now, or None. A backup going
+    doesn't stop one: the test only reads this install's database."""
+    if restore_going(session):
+        return "a restore is going"
+    going = session.execute(
+        select(BackupRun.id).where(BackupRun.kind == "test")
+        .where(BackupRun.status == "running").limit(1)
+    ).first()
+    return "a test restore is already running" if going else None
+
+
+def end_cut_off_tests() -> int:
+    """End every test restore a restart cut short, and drop the temporary
+    database it left. Called when the long-maintenance lane starts, where the
+    tests run, so none is alive then."""
+    with _sync_session_factory()() as session:
+        ended = session.execute(
+            update(BackupRun).where(BackupRun.kind == "test")
+            .where(BackupRun.status == "running")
+            .values(status="error", finished_at=datetime.now(UTC),
+                    error="Interrupted: the server restarted during the test. Test again.")
+        ).rowcount or 0
+        session.commit()
+    if ended:
+        backup_verify.drop_scratch(get_config().database_url_sync)
+    return ended
+
+
+def _library_is_empty(session) -> bool:
+    return session.execute(text(
+        "SELECT NOT EXISTS (SELECT 1 FROM image_record) "
+        "AND NOT EXISTS (SELECT 1 FROM post_attachment)"
+    )).scalar()
+
+
+def _note(session, row: BackupRun, m: dict, token: str | None = None) -> bool:
+    """Write the run's progress. Given a chunk's token, writes only while that
+    chunk still carries the run, and returns False when it doesn't."""
+    if token is not None and not _still_ours(session, row.id, token):
+        return False
+    row.manifest = dict(m)
+    row.last_progress_at = datetime.now(UTC)
+    session.commit()
+    return True
+
+
+@celery.task(
+    name="backend.app.tasks.backup.restore_offsite_task",
+    bind=True,
+    max_retries=0,  # a half-applied restore is never retried on its own
+    soft_time_limit=21600, time_limit=23400,
+)
+def restore_offsite_task(self, *, dump_key: str | None = None, files_only: bool = False,
+                         run_id: int | None = None, after: str | None = None,
+                         resume: bool = False) -> dict:
+    """Restore this install from the off-site bucket, one chunk at a time.
+
+    Started without `run_id`, it replaces the database with the dump at
+    `dump_key` (the newest when None), then fetches every library file the
+    restored rows name. With `files_only`, the database stays and only the
+    missing files are fetched, which also repairs a library that lost files.
+    Each chunk queues the next, and a restart resumes the file phase where it
+    stopped; the run's row counts files restored from the bucket, copied from
+    a duplicate already on disk, already present, missing from the bucket,
+    and failed. Returns {'backup_run_id': N, ...}."""
+    SessionLocal = _sync_session_factory()
+    token = _chunk_token(self)
+    if run_id is not None:
+        return _restore_files(SessionLocal, run_id, token, after=after, resume=resume)
+
+    now = datetime.now(UTC)
+    with SessionLocal() as session:
+        busy = backup_busy(session)
+        if busy:
+            return {"skipped": busy}
+        settings = ImportSettings.load_sync(session)
+        limit_mbps = settings.backup_upload_limit_mbps
+        row = BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                        started_at=now, last_progress_at=now, size_bytes=0,
+                        manifest={"phase": "files" if files_only else "database",
+                                  "cursor": "", "failures": [],
+                                  **dict.fromkeys(_RESTORE_COUNTS, 0)})
+        session.add(row)
+        session.commit()
+        run_id = row.id
+        try:
+            target = backup_destination.saved_target(settings)
+            if target is None:
+                raise backup_s3.S3Error("No off-site destination is set up.")
+        except backup_s3.S3Error as exc:
+            _mark_failed(session, row, exc)
+            raise
+        row.manifest = {**row.manifest, "offsite": _offsite_manifest(target)}
+        session.commit()
+
+    if files_only:
+        return _restore_files(SessionLocal, run_id, token, after=None, resume=False)
+    run_id = _restore_database(SessionLocal, run_id, target, limit_mbps, dump_key)
+    # The files follow as chunks of their own, on the worker.
+    restore_offsite_task.delay(run_id=run_id)
+    return {"backup_run_id": run_id, "continues": True}
+
+
+def _restore_database(SessionLocal, run_id: int, target, limit_mbps, dump_key) -> int:
+    """Replace the database with a dump from the bucket. Returns the run's id
+    in the restored database, where its row is written again."""
+    cfg = get_config()
+    pacer = backup_s3.Pacer(limit_mbps)
+    staged = None
+    try:
+        dump = backup_restore.find_dump(target, dump_key)
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            m = dict(row.manifest)
+            m["dump"] = {"key": dump.key, "ts": dump.ts, "size_bytes": dump.size_bytes}
+            # The database about to be replaced goes to the bucket first, so
+            # the restore can itself be undone. An empty one isn't worth a
+            # dump, and would sit in the list as the newest.
+            if not _library_is_empty(session):
+                session.commit()
+                saved = _dump_offsite(cfg, target, limit_mbps)
+                m["saved_before"] = saved["key"]
+            _note(session, row, m)
+            started_at, manifest = row.started_at, dict(m)
+        staged = backup_restore.fetch_dump(target, dump, IMAGES_ROOT / "_backups", pacer)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        if staged is not None:
+            staged.unlink(missing_ok=True)
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            if row is not None:
+                _mark_failed(session, row, exc)
+        raise
+
+    # From here on the database is being replaced; every pooled connection
+    # still points at the tables being dropped.
+    get_sync_engine().dispose()
+    try:
+        backup_service.restore_db(db_url=cfg.database_url_sync, sql_path=staged)
+        backup_restore.migrate()
+        backup_service.end_other_connections(db_url=cfg.database_url_sync)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        _leave_empty_install(SessionLocal, cfg, target, limit_mbps, started_at, manifest, exc)
+        raise
+    finally:
+        staged.unlink(missing_ok=True)
+    get_sync_engine().dispose()
+
+    with SessionLocal() as session:
+        backup_restore.keep_destination(session, target, limit_mbps)
+        # The dump holds its own run as still going: it is the dump just
+        # restored, so it finished. Any other run going then is over.
+        dump_at = datetime.strptime(manifest["dump"]["ts"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
+        own = session.execute(
+            select(BackupRun).where(BackupRun.kind == "db", BackupRun.status == "running")
+            # The name's time is truncated to the second, after the row began.
+            .where(BackupRun.started_at.between(dump_at - timedelta(minutes=2),
+                                                dump_at + timedelta(seconds=1)))
+            .order_by(BackupRun.started_at.desc()).limit(1)
+        ).scalar_one_or_none()
+        if own is not None:
+            own.status = "ok"
+            own.manifest = {"ts": manifest["dump"]["ts"], "offsite": {
+                **manifest["offsite"], "key": manifest["dump"]["key"]}}
+            own.size_bytes = manifest["dump"]["size_bytes"]
+        session.execute(
+            update(BackupRun).where(BackupRun.status.in_(["running", "restoring"]))
+            .values(status="error", finished_at=datetime.now(UTC),
+                    error="Interrupted: this run was going when the restored dump was taken.")
+        )
+        row = BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                        started_at=started_at, last_progress_at=datetime.now(UTC),
+                        size_bytes=0, manifest={**manifest, "phase": "files"})
+        session.add(row)
+        session.commit()
+        return row.id
+
+
+def _leave_empty_install(SessionLocal, cfg, target, limit_mbps, started_at, manifest,
+                         exc: BaseException) -> None:
+    """After a failed load, put back an empty database at this version, so the
+    install starts and can try again, and record why it stopped."""
+    try:
+        backup_service.wipe_db(db_url=cfg.database_url_sync)
+        backup_restore.migrate()
+        backup_service.end_other_connections(db_url=cfg.database_url_sync)
+        get_sync_engine().dispose()
+        with SessionLocal() as session:
+            backup_restore.keep_destination(session, target, limit_mbps)
+            session.add(BackupRun(
+                kind="restore", status="error", triggered_by="restore",
+                started_at=started_at, finished_at=datetime.now(UTC), manifest=manifest,
+                error=(f"The restore stopped while loading the database: {exc}. "
+                       "The database is empty again; choose a dump and restore again.")[:2000],
+            ))
+            session.commit()
+    except Exception:  # noqa: BLE001 — the original failure is the one raised
+        log.exception("restore: the database couldn't be reset after a failed load")
+
+
+def _restore_files(SessionLocal, run_id: int, token: str, *, after: str | None,
+                   resume: bool) -> dict:
+    """One chunk of a restore's file phase; queues the next while files remain."""
+    with SessionLocal() as session:
+        why = _claim_run(session, run_id, "restoring", token, after=after, resume=resume)
+        if why:
+            return {"backup_run_id": run_id, "skipped": why}
+        row = session.get(BackupRun, run_id)
+        settings = ImportSettings.load_sync(session)
+        limit_mbps = settings.backup_upload_limit_mbps
+        try:
+            target = backup_destination.saved_target(settings)
+            if target is None:
+                raise backup_s3.S3Error("No off-site destination is set up.")
+        except backup_s3.S3Error as exc:
+            _mark_failed(session, row, exc)
+            raise
+    try:
+        more = _fetch_files_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps),
+                                  token)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            if row is not None:
+                _mark_failed(session, row, exc)
+        raise
+
+    if more is None:
+        return {"backup_run_id": run_id, "skipped": "another chunk took the run over"}
+    if more:
+        restore_offsite_task.delay(run_id=run_id, after=token)
+        return {"backup_run_id": run_id, "continues": True}
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        row.status = "restored"
+        row.finished_at = datetime.now(UTC)
+        missed = m.get("missing", 0) + m.get("failed", 0)
+        if missed:
+            first = (m.get("failures") or [{}])[0]
+            row.error = (f"{m.get('missing', 0)} file(s) aren't in the bucket and "
+                         f"{m.get('failed', 0)} couldn't be restored. "
+                         f"First: {first.get('path')}: {first.get('error')}")[:2000]
+        session.commit()
+    # Thumbnails aren't backed up: they are made again from the restored files.
+    from .thumbnail import backfill_thumbnails
+    backfill_thumbnails.delay()
+    return {"backup_run_id": run_id, "continues": False}
+
+
+def _fetch_files_chunk(SessionLocal, run_id: int, target, pacer, token: str) -> bool | None:
+    """Put library files back until the chunk's time is up or none are left.
+    Returns True when files remain for another chunk, and None when another
+    chunk has taken the run over."""
+    tid = backup_media.target_id(target)
+    deadline = time.monotonic() + RESTORE_CHUNK_SECONDS
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        cursor = m.get("cursor") or ""
+        noted = time.monotonic()
+        while True:
+            batch = backup_restore.library_files(session, after=cursor)
+            keys = backup_restore.sent_keys(session, tid, [f.sha256 for f in batch])
+            # End the read before any download starts.
+            session.commit()
+            if not batch:
+                m["cursor"] = cursor
+                return False if _note(session, row, m, token) else None
+            for f in batch:
+                if time.monotonic() > deadline:
+                    m["cursor"] = cursor
+                    return True if _note(session, row, m, token) else None
+                try:
+                    outcome = backup_restore.restore_file(
+                        target, f, keys.get(f.sha256), IMAGES_ROOT, pacer)
+                except (backup_s3.S3Error, OSError) as exc:
+                    outcome = "missing" if isinstance(exc, backup_s3.ObjectMissing) else "failed"
+                    if len(m.get("failures") or []) < RESTORE_FAILURES_LISTED:
+                        m["failures"] = [*(m.get("failures") or []),
+                                         {"path": f.paths[0], "error": str(exc)[:300]}]
+                m[outcome] = m.get(outcome, 0) + 1
+                if outcome in ("restored", "copied"):
+                    row.size_bytes = (row.size_bytes or 0) + f.size_bytes
+                cursor = f.sha256
+                m["cursor"] = cursor
+                if outcome != "present" or time.monotonic() - noted > _RESTORE_NOTE_SECONDS:
+                    if not _note(session, row, m, token):
+                        return None
+                    noted = time.monotonic()

backend/app/tasks/external.py

@@ -24,7 +24,7 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 import redis
-from sqlalchemy import delete, select, update
+from sqlalchemy import delete, or_, select, update
 
 from ..celery_app import celery
 from ..config import get_config
@@ -32,6 +32,7 @@ from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, So
 from ..services.external_fetch import fetch_external
 from ..services.importer import Importer
 from ..services.thumbnailer import Thumbnailer
+from ..utils.staging import is_temp_staging_file
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
 log = logging.getLogger(__name__)
@@ -39,14 +40,20 @@ log = logging.getLogger(__name__)
 IMAGES_ROOT = Path("/images")
 
 # After this many failed attempts a link is dead-lettered (skipped by routine
-# sweeps; an operator recovery still re-attempts). Mirrors the ingester ledger.
+# sweeps; the daily recovery retries it, sparingly). Mirrors the ingester ledger.
 DEAD_LETTER_THRESHOLD = 3
 # Per-fetch read + total budgets now live in external_fetch (a short read
 # timeout fails a stalled host fast; a generous total caps a big-but-flowing
 # download). The celery soft/hard time_limit is the outer backstop above those.
 # Links enqueued per sweep — bounds the burst when a big backfill records many.
 _SWEEP_BATCH = 50
-# Dead rows older than this are pruned (retention).
+# Recovery gives a dead link one more attempt once it has been dead for
+# 2**recoveries days (1, 2, 4, 8, 16), and stops after _MAX_RECOVERIES. A host
+# outage gets several later chances; a link that can never succeed (too big to
+# import in time, a file made private) costs a handful of downloads, not three
+# a day.
+_MAX_RECOVERIES = 5
+# A link recovery has given up on is pruned this long after its last attempt.
 _RETENTION_DAYS = 30
 # Per-host serialize lock. TTL is a safety net for a worker that dies holding it
 # (normal completion/error releases it in `finally`); sized just past the fetch
@@ -54,6 +61,9 @@ _RETENTION_DAYS = 30
 # than one fetch would have taken.
 _LOCK_PREFIX = "fc:extdl_lock:"
 _LOCK_TTL = 2400
+# A link 'downloading' this long has outlived fetch_external_link's hard limit
+# (time_limit=3600), so its worker is gone and the sweep recovers it.
+_STALL_MINUTES = 65
 _SERIALIZE_COUNTDOWN = 120
 _MAX_SERIALIZE_WAITS = 30
 
@@ -106,7 +116,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
                 ExternalLink.id == link_id,
                 ExternalLink.status.in_(("pending", "failed")),
             )
-            .values(status="downloading")
+            .values(status="downloading", started_at=datetime.now(UTC))
             .returning(ExternalLink.id)
         ).first()
         session.commit()
@@ -180,6 +190,9 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
         / str(post.external_post_id) / str(link_id)
     )
     try:
+        # Clear the staging files an interrupted attempt left behind. Complete
+        # files it left stay, and are imported once this fetch succeeds.
+        _discard_partials(post_dir)
         result = fetch_external(host, url, post_dir)
         with SessionLocal() as session:
             link = session.get(ExternalLink, link_id)
@@ -190,6 +203,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
                 )
                 _record_failure(link, attempts, result.error or "fetch failed")
                 session.commit()
+                _discard_partials(post_dir)
                 return {"link_id": link_id, "error": result.error}
 
             log.info(
@@ -201,7 +215,8 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             post = session.get(Post, link.post_id)
             artist = session.get(Artist, post.artist_id)
             source = session.get(Source, post.source_id) if post.source_id else None
-            image_ids = _route_files(session, result.files, post, platform, artist, source)
+            files = _unimported_files(session, post_dir, first=result.files)
+            image_ids = _route_files(session, files, post, platform, artist, source)
             link.status = "downloaded"
             link.last_error = None
             link.completed_at = datetime.now(UTC)
@@ -231,6 +246,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             if link is not None and link.status == "downloading":
                 _record_failure(link, attempts, f"{type(exc).__name__}: {exc}")
                 session.commit()
+        _discard_partials(post_dir)
         raise
     finally:
         if lock is not None:
@@ -287,10 +303,9 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
                 post.id, f.name, result.status, len(ids),
             )
         elif result.status == "attached":
-            # Non-art / archive captured as a PostAttachment (copied into the
-            # store) — drop the on-disk original. An archive captured WITHOUT
-            # extracting any image carries the reason (the recurring "zip but no
-            # images" symptom, #718) — surface it loudly.
+            # Non-art, or an archive with no new images, captured as a
+            # PostAttachment. An archive captured WITHOUT extracting any image
+            # carries the reason, so the "zip but no images" case is visible.
             if result.error:
                 log.warning(
                     "extdl archive captured UNEXTRACTED: post=%s file=%s reason=%s",
@@ -298,51 +313,104 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
                 )
             else:
                 log.info("extdl attachment captured: post=%s file=%s", post.id, f.name)
-            f.unlink(missing_ok=True)
         elif result.status == "skipped":
             reason = result.skip_reason.value if result.skip_reason else None
             log.info(
                 "extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason,
             )
-            if reason in ("duplicate_hash", "duplicate_phash"):
-                # Path-safe unlink: external files are imported IN PLACE, so `f`
-                # can BE the canonical record's backing file (same content
-                # re-fetched / a colliding name). Only delete `f` when it is NOT
-                # the existing record's file — otherwise we orphan the record and
-                # playback 404s. (#859)
-                canonical = None
-                if result.image_id is not None:
-                    rec = session.get(ImageRecord, result.image_id)
-                    if rec is not None:
-                        canonical = (IMAGES_ROOT / rec.path).resolve()
-                if canonical != f.resolve():
-                    f.unlink(missing_ok=True)  # a redundant copy — safe to drop
-                else:
-                    log.info(
-                        "extdl keep: %s IS record %s's canonical file — not unlinking",
-                        f.name, result.image_id,
-                    )
         elif result.status == "failed":
             log.warning(
                 "extdl import FAILED: post=%s file=%s error=%s",
                 post.id, f.name, result.error,
             )
-            f.unlink(missing_ok=True)
         else:  # 'refreshed' or any future status — work happened, no new images
             log.info(
                 "extdl import: post=%s file=%s status=%s", post.id, f.name, result.status,
             )
+        # Same policy as the subscription download path: drop the file unless
+        # the library needs it at this path. A file imported in place is kept.
+        importer.discard_after_attach(f, result)
         # The synthesized sidecar has done its job — don't litter the tree.
         Path(str(f) + ".json").unlink(missing_ok=True)
     return image_ids
 
 
+def _unimported_files(session, link_dir: Path, *, first=()) -> list[Path]:
+    """Every complete file under a link's dir, subfolders included, that no
+    ImageRecord points at: what this fetch wrote plus anything an earlier
+    attempt finished writing but never imported. `first` (the fetch's own
+    files) leads the list. Staging files and sidecars are left out."""
+    if not link_dir.is_dir():
+        return []
+    found = {
+        p for p in link_dir.rglob("*")
+        if p.is_file() and not is_temp_staging_file(p) and p.suffix.lower() != ".json"
+    }
+    ordered = [p for p in first if p in found] + sorted(found.difference(first))
+    recorded = set(session.execute(
+        select(ImageRecord.path).where(
+            ImageRecord.path.in_([str(p) for p in ordered])
+        )
+    ).scalars()) if ordered else set()
+    return [p for p in ordered if str(p) not in recorded]
+
+
+def _discard_partials(link_dir: Path) -> int:
+    """Delete the staging files under a link's dir (megatools' `.megatmp.*`, an
+    HTTP fetch's `.part`), then the dirs left empty. Every fetcher writes to a
+    staging name and renames on completion, so a staging file is a transfer
+    that never finished. Complete files stay: they may be the only copy of art
+    whose upstream has since gone. Returns the number of files removed."""
+    if not link_dir.is_dir():
+        return 0
+    removed = 0
+    for p in link_dir.rglob("*"):
+        if not (p.is_file() and is_temp_staging_file(p)):
+            continue
+        try:
+            p.unlink(missing_ok=True)
+            removed += 1
+        except OSError as exc:
+            log.warning("extdl could not remove %s: %s", p, exc)
+    for d in sorted((p for p in link_dir.rglob("*") if p.is_dir()), reverse=True):
+        try:
+            d.rmdir()
+        except OSError:
+            pass  # not empty: it holds a complete file
+    try:
+        link_dir.rmdir()
+    except OSError:
+        pass
+    if removed:
+        log.info("extdl removed %d leftover file(s) from %s", removed, link_dir)
+    return removed
+
+
 @celery.task(name="backend.app.tasks.external.sweep_external_links")
 def sweep_external_links() -> dict:
     """Enqueue a bounded batch of actionable links (pending, or failed below the
-    dead-letter threshold). Driven by the post-download hook and a beat tick."""
+    dead-letter threshold). Driven by the post-download hook and a beat tick.
+
+    First it recovers links whose worker died mid-fetch: a link 'downloading'
+    for longer than the task's hard limit has no worker left, so it counts as a
+    failed attempt and goes back to the retry path (dead-lettered at the
+    threshold like any other failure). The next attempt clears its staging
+    files and imports the complete ones."""
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
+        now = datetime.now(UTC)
+        stale = session.execute(
+            select(ExternalLink).where(
+                ExternalLink.status == "downloading",
+                or_(
+                    ExternalLink.started_at.is_(None),
+                    ExternalLink.started_at < now - timedelta(minutes=_STALL_MINUTES),
+                ),
+            )
+        ).scalars().all()
+        for link in stale:
+            _record_failure(link, link.attempts, "worker stopped mid-fetch")
+        session.commit()
         ids = session.execute(
             select(ExternalLink.id).where(
                 ExternalLink.status.in_(("pending", "failed")),
@@ -351,34 +419,53 @@ def sweep_external_links() -> dict:
         ).scalars().all()
     for link_id in ids:
         fetch_external_link.delay(link_id)
-    return {"enqueued": len(ids)}
+    return {"enqueued": len(ids), "recovered": len(stale)}
 
 
 @celery.task(name="backend.app.tasks.external.recover_external_links")
 def recover_external_links() -> dict:
-    """Recovery sweep (rule #89): reset dead links back to retryable so a stuck
-    host outage or a since-fixed bug gets another pass, then enqueue. Bounded."""
+    """Recovery sweep: give each dead link one more attempt once it
+    has been dead for 2**recoveries days, then enqueue. A recovered link that
+    fails again goes straight back to dead, and after _MAX_RECOVERIES it stays
+    dead until retention prunes it."""
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
-        session.execute(
-            update(ExternalLink).where(ExternalLink.status == "dead")
-            .values(status="failed", attempts=0, last_error=None)
-        )
+        now = datetime.now(UTC)
+        candidates = session.execute(
+            select(ExternalLink).where(
+                ExternalLink.status == "dead",
+                ExternalLink.recoveries < _MAX_RECOVERIES,
+            )
+        ).scalars().all()
+        recovered = 0
+        for link in candidates:
+            wait = timedelta(days=2 ** link.recoveries)
+            if link.completed_at is not None and link.completed_at > now - wait:
+                continue
+            link.status = "failed"
+            link.attempts = DEAD_LETTER_THRESHOLD - 1
+            link.recoveries += 1
+            recovered += 1
         session.commit()
-    return sweep_external_links()
+    return {**sweep_external_links(), "revived": recovered}
 
 
 @celery.task(name="backend.app.tasks.external.prune_external_links")
 def prune_external_links() -> dict:
-    """Retention (rule #89): delete long-dead links so the ledger doesn't grow
-    unbounded. Downloaded links are kept (they're the record of what we have)."""
+    """Retention: delete links recovery has given up on, once their
+    last attempt is _RETENTION_DAYS old, so the ledger doesn't grow unbounded.
+    Downloaded links are kept (they're the record of what we have)."""
     cutoff = datetime.now(UTC) - timedelta(days=_RETENTION_DAYS)
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
         res = session.execute(
             delete(ExternalLink).where(
                 ExternalLink.status == "dead",
-                ExternalLink.created_at < cutoff,
+                ExternalLink.recoveries >= _MAX_RECOVERIES,
+                or_(
+                    ExternalLink.completed_at.is_(None),
+                    ExternalLink.completed_at < cutoff,
+                ),
             )
         )
         session.commit()

backend/app/tasks/import_file.py

@@ -19,7 +19,8 @@ from sqlalchemy.exc import DBAPIError, OperationalError
 
 from ..celery_app import celery
 from ..models import ImportBatch, ImportSettings, ImportTask
-from ..services.importer import Importer
+from ..services.importer import Importer, ImportResult
+from ..services.library_scan import folder_owner
 from ..services.thumbnailer import Thumbnailer
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -45,8 +46,13 @@ def _mark_failed(session, task, error_msg: str) -> None:
     """Best-effort flip of a 'processing' row to 'failed' + batch counter
     increment. Wrapped in its own try because if the DB is what just
     broke, this commit will also fail — that's why the maintenance sweep
-    exists as a backstop."""
+    exists as a backstop.
+
+    The failure being recorded may have left the session's transaction dead
+    (an error mid-flush), so it is rolled back first; otherwise this commit
+    fails on it too and the row stays 'processing'."""
     try:
+        session.rollback()
         task.status = "failed"
         task.error = error_msg
         task.finished_at = datetime.now(UTC)
@@ -165,6 +171,27 @@ def enqueue_import(task_id: int, task_type: str) -> None:
         import_media_file.delay(task_id)
 
 
+def _attach_where_it_lies(session, importer: Importer, path: Path) -> ImportResult:
+    """Import a file that already sits in its artist's folder (a `library`
+    batch, from services.library_scan). It is attached in place with that
+    artist and source. The post record its folder holds (`_post.json`) is filed
+    first, so the post gets its title and date. Afterwards the file is removed
+    if the library didn't keep it: a copy of something it already holds, or a
+    file the import filters reject. A file whose import crashed stays, with the
+    task marked failed: it may be the only copy of an archive the extractor
+    can't read yet."""
+    if not path.is_file():
+        return ImportResult(status="skipped", error="no longer on disk")
+    artist, source = folder_owner(session, path, IMAGES_ROOT)
+    record = path.parent / "_post.json"
+    if artist is not None and record.is_file():
+        importer.upsert_post_record(record, artist=artist, source=source)
+    result = importer.attach_in_place(path, artist=artist, source=source)
+    if result.status != "failed":
+        importer.discard_after_attach(path, result)
+    return result
+
+
 def _do_import(session, task, import_task_id: int) -> dict:
     """Actual work, called from inside the resilience wrapper."""
     settings = ImportSettings.load_sync(session)
@@ -180,7 +207,10 @@ def _do_import(session, task, import_task_id: int) -> dict:
         deep=deep,
     )
 
-    result = importer.import_one(Path(task.source_path))
+    if batch is not None and batch.scan_mode == "library":
+        result = _attach_where_it_lies(session, importer, Path(task.source_path))
+    else:
+        result = importer.import_one(Path(task.source_path))
 
     if result.status in ("imported", "superseded"):
         task.status = "complete"

backend/app/tasks/maintenance.py

@@ -20,10 +20,12 @@ from ..models import (
     ImportSettings,
     ImportTask,
     LibraryAuditRun,
+    PostAttachment,
     Source,
     TaskRun,
 )
 from ..utils.phash import compute_phash
+from ..utils.staging import is_temp_staging_file
 from ._sync_engine import get_sync_engine
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -81,13 +83,10 @@ WIP_BACKFILL_PAGE = 500
 FFPROBE_TIMEOUT_SECONDS = 10
 TASK_RUN_KEEP_OK_SECONDS = 24 * 3600       # 24 h
 TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600  # 7 days
-# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
-# then os.replace() into place (importer / external_fetch / native_ingest_common /
-# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
-# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
-# keeps it from deleting an in-flight download's staging file mid-write.
+# Orphaned staging files (utils.staging): cleanup_orphaned_temp_files sweeps the
+# ones a killed write left behind; the min-age guard keeps it from deleting an
+# in-flight download's staging file mid-write.
 IMAGES_ROOT = Path("/images")
-TEMP_STAGING_SUFFIXES = (".part", ".partial")
 ORPHAN_TEMP_MIN_AGE_HOURS = 6
 
 # Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
@@ -137,7 +136,13 @@ IMPORT_BATCH_KEEP_DAYS = 30
 #   (the import queue itself stays at the 5-min default for single
 #   files); time_limit=2100.
 QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
-    "ml": 25,
+    # ml: the scheduled auto-apply sweeps and refresh_character_prototypes run
+    # to a 35-min hard limit (2100s); 25 swept them mid-run (#4432). The two
+    # 65-min jobs have their own entries below.
+    "ml": 40,
+    # import: import_media_file's hard limit is 6 min (360s), one past the
+    # 5-min default this queue fell to (#4432).
+    "import": 10,
     # download_source legitimately walks 5-25 min (Patreon/gallery-dl
     # deep creators); its hard time_limit is DOWNLOAD_HARD_TIME_LIMIT
     # (1500s = 25m). The 5-min default flagged healthy in-flight walks as
@@ -154,14 +159,23 @@ QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # overrides below cover the outliers (backups, library audit).
     "maintenance": 75,
     "scan": 75,
+    # The long lane (#4432). Until TaskRun.queue asked the router, nothing was
+    # recorded here: these runs read as `maintenance` (75) or, for
+    # translation, `default` (5 — which failed healthy 35-min runs). The
+    # longest task without its own entry below is the admin family at a
+    # 40-min hard limit; 45 = 40 + 5.
+    "maintenance_long": 45,
 }
 TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     "backend.app.tasks.import_file.import_archive_file": 40,
-    # Backup images runs hours, not minutes (6.5h hard limit). The
-    # task-name override beats the queue's 75-min default so a
-    # legitimately-running backup isn't flagged.
-    "backend.app.tasks.backup.backup_images_task": 420,
-    "backend.app.tasks.backup.restore_images_task": 420,
+    # One media-backup chunk stops starting files after 45 min but may finish
+    # a large file in flight, up to its 6.5 h hard limit.
+    "backend.app.tasks.backup.backup_media_task": 420,
+    # A restore's database load, or one chunk of its file phase, likewise.
+    "backend.app.tasks.backup.restore_offsite_task": 420,
+    # A test restore downloads a dump and loads it whole; 4 h 15 min hard
+    # limit. 260 = 255 + 5.
+    "backend.app.tasks.backup.restore_test_task": 260,
     # Library audit scans the full library — 2h hard limit.
     "backend.app.tasks.library_audit.scan_library_for_rule": 130,
     # External file-host fetches (mega/gdrive/film packs) can run to the task's
@@ -179,9 +193,33 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # external-fetch entry above — without an override a healthy in-flight walk
     # is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
     "backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
+    # Importing what's on disk stats every file in the artists' folders, and
+    # its hard limit is 40 min for the same wedged-filesystem case. 45 = 40 + 5.
+    "backend.app.tasks.admin.import_unimported_files_task": 45,
+    # Head training and the manual head apply run to 65 min (3900s) — past the
+    # ml queue's threshold (#4432). 70 = 65 + 5.
+    "backend.app.tasks.ml.train_heads": 70,
+    "backend.app.tasks.ml.apply_head_tags": 70,
 }
 
 
+def _import_queue_depth() -> int | None:
+    """Messages waiting in the broker's import queue, or None when the broker
+    can't be read."""
+    try:
+        import redis
+
+        from ..config import get_config
+
+        client = redis.Redis.from_url(
+            get_config().celery_broker_url, socket_timeout=5, socket_connect_timeout=5,
+        )
+        return int(client.llen("import"))
+    except Exception:  # noqa: BLE001 — an unreadable broker means "don't know"
+        log.warning("recover_interrupted_tasks: import queue depth unreadable", exc_info=True)
+        return None
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
 def recover_interrupted_tasks() -> int:
     """Recover stuck ImportTask rows. Two distinct stuck states:
@@ -197,16 +235,22 @@ def recover_interrupted_tasks() -> int:
        2026-05-24 after a 2224-row zombie pile; archive split out
        2026-05-28.)
 
-    2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
-       creates rows with status='pending' (commit), then in a second pass
-       transitions to 'queued' and calls .delay() (commit). If the scanner
-       crashes between those two commits, rows are orphaned in 'pending'
-       (never enqueued) with no recovery path — invisible to the
-       'processing' sweep above. Flagged 2026-05-25 by operator hitting a
-       5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
-       the operator drains them via /api/import/retry-failed at their own
-       pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
-       import worker.
+    2. 'pending' > 30 min — enqueue-phase crash. scan_directory creates rows
+       with status='pending' (commit), then in a second pass transitions to
+       'queued' and calls .delay() (commit). If the scanner crashes between
+       those two commits, rows are orphaned in 'pending' (never enqueued)
+       with no recovery path — invisible to the 'processing' sweep above.
+       Flip these to 'failed' (not re-enqueue) so the operator drains them
+       via /api/import/retry-failed at their own pace; bulk-re-enqueueing
+       5000+ rows would thundering-herd the import worker.
+
+       A 'queued' row has a broker message, and in a batch larger than half
+       an hour of work it is simply waiting its turn, so its age says
+       nothing. It counts as orphaned only when the broker's import queue
+       is empty: then its message is gone (a broker flush or restart) and
+       nothing will ever run it. When the queue can't be read, queued rows
+       are left alone — a queued row costs nothing to keep, and failing a
+       live one loses its import.
 
     Returns total rows touched (recovered + marked failed).
     """
@@ -283,9 +327,12 @@ def recover_interrupted_tasks() -> int:
         )
         stuck = stuck_result.all()
 
+        orphan_statuses = ["pending"]
+        if _import_queue_depth() == 0:
+            orphan_statuses.append("queued")
         orphan_result = session.execute(
             update(ImportTask)
-            .where(ImportTask.status.in_(["pending", "queued"]))
+            .where(ImportTask.status.in_(orphan_statuses))
             .where(ImportTask.created_at < orphan_cutoff)
             .values(
                 status="failed",
@@ -355,22 +402,58 @@ def cleanup_old_tasks() -> int:
         return result.rowcount or 0
 
 
+def _paths_with_rows(paths: list[str]) -> set[str]:
+    """The subset of `paths` that an image (its file or thumbnail) or an
+    attachment row points at."""
+    owned: set[str] = set()
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        for i in range(0, len(paths), 500):
+            chunk = paths[i:i + 500]
+            owned.update(session.execute(
+                select(ImageRecord.path).where(ImageRecord.path.in_(chunk))
+            ).scalars())
+            owned.update(session.execute(
+                select(ImageRecord.thumbnail_path).where(ImageRecord.thumbnail_path.in_(chunk))
+            ).scalars())
+            owned.update(session.execute(
+                select(PostAttachment.path).where(PostAttachment.path.in_(chunk))
+            ).scalars())
+    return owned
+
+
 @celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
 def cleanup_orphaned_temp_files() -> int:
-    """Delete orphaned .part/.partial staging files under the images root, left by
-    a download/import killed mid-write. Only removes files older than
-    ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
-    pulled out from under it. Returns the count removed."""
+    """Delete orphaned staging files (.part, .partial, megatools' .megatmp.*)
+    under the images root, left by a download/import killed mid-write. Only
+    removes files older than ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight
+    download's staging file is never pulled out from under it, and never a
+    file an image or attachment row points at, whatever its name: that file
+    is library content. Returns the count removed."""
     if not IMAGES_ROOT.is_dir():
         return 0
     cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
-    removed = 0
+    stale: list[Path] = []
     for path in IMAGES_ROOT.rglob("*"):
-        if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
+        if not is_temp_staging_file(path) or not path.is_file():
+            continue
+        try:
+            if path.stat().st_mtime < cutoff:   # fresher may be an active download
+                stale.append(path)
+        except OSError as exc:
+            log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
+    if not stale:
+        return 0
+    owned = _paths_with_rows([str(p) for p in stale])
+    removed = 0
+    for path in stale:
+        if str(path) in owned:
+            log.warning(
+                "cleanup_orphaned_temp_files: keeping %s: a library row points at it",
+                path,
+            )
             continue
         try:
-            if path.stat().st_mtime >= cutoff:
-                continue          # too fresh — may be an active download
             path.unlink()
             removed += 1
         except OSError as exc:
@@ -473,6 +556,10 @@ def prune_task_runs() -> dict:
       (recover_stalled_task_runs) is the mechanism that flips them to
       terminal state; prune doesn't touch in-flight state.
     - 'retry' rows: treated as failures (>7d).
+    - The NEWEST row of each task is never deleted, whatever its age: it is
+      what the beat scheduler reads to know when a job last ran (#4408).
+      Without it a weekly job's last run would be pruned after a day, and beat
+      would think it had never run and fire it on every restart.
 
     Returns dict of how many rows were deleted in each bucket.
     """
@@ -480,16 +567,19 @@ def prune_task_runs() -> dict:
     now = datetime.now(UTC)
     ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
     fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
+    newest = select(func.max(TaskRun.id)).group_by(TaskRun.task_name)
     with SessionLocal() as session:
         ok_deleted = session.execute(
             delete(TaskRun)
             .where(TaskRun.status == "ok")
             .where(TaskRun.finished_at < ok_cutoff)
+            .where(TaskRun.id.not_in(newest))
         ).rowcount or 0
         fail_deleted = session.execute(
             delete(TaskRun)
             .where(TaskRun.status.in_(["error", "timeout", "retry"]))
             .where(TaskRun.finished_at < fail_cutoff)
+            .where(TaskRun.id.not_in(newest))
         ).rowcount or 0
         session.commit()
         return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
@@ -708,6 +798,32 @@ def recover_stalled_download_events() -> int:
         return events_recovered
 
 
+@celery.task(
+    name="backend.app.tasks.maintenance.date_posts_from_records",
+    soft_time_limit=1500,
+    time_limit=1800,
+)
+def date_posts_from_records() -> dict:
+    """Date undated native posts from the records their walk left on disk
+    (#4436). Hourly and self-limiting: once every post is dated it is one
+    empty query."""
+    from ..services.importer import Importer
+    from ..services.post_record_repair import date_posts_from_records as _repair
+    from ..services.thumbnailer import Thumbnailer
+
+    images_root = IMAGES_ROOT
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        importer = Importer(
+            session=session,
+            images_root=images_root,
+            import_root=images_root,
+            thumbnailer=Thumbnailer(images_root=images_root),
+            settings=ImportSettings.load_sync(session),
+        )
+        return _repair(session, importer, images_root)
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
 def recover_stalled_backup_runs() -> int:
     """Flip BackupRun rows stuck in running/restoring past the hard limit
@@ -728,11 +844,20 @@ def recover_stalled_backup_runs() -> int:
         result = session.execute(
             update(BackupRun)
             .where(BackupRun.status.in_(["running", "restoring"]))
-            # db backups/restores are fast (40-min window); images run hours (7h).
+            # db backups/restores are fast (40-min window); a test restore and the
+            # weekly bucket check get 7 h.
+            # A media run or an off-site restore spans many chunks and hours on
+            # purpose, so its window runs from its last progress, not its start.
             .where(
                 or_(
                     and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff),
-                    and_(BackupRun.kind != "db", BackupRun.started_at < slow_cutoff),
+                    and_(
+                        BackupRun.kind.in_(["media", "restore"]),
+                        func.coalesce(BackupRun.last_progress_at, BackupRun.started_at)
+                        < slow_cutoff,
+                    ),
+                    and_(BackupRun.kind.not_in(["db", "media", "restore"]),
+                         BackupRun.started_at < slow_cutoff),
                 )
             )
             .values(status="error", finished_at=now, error=msg)
@@ -1034,8 +1159,7 @@ def cleanup_old_download_events() -> int:
 
 def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
     """One keyset-paginated pass over posts whose title matches a WIP tier, applying
-    `tag_id` (stamped `source`) to their images. Shared by the hard + soft tiers
-    (#1458 / #1474). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
+    `tag_id` (stamped `source`) to their images (#1458). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
     `matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
     count newly applied."""
     from ..models import Post
@@ -1075,26 +1199,17 @@ def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
 )
 def backfill_wip_title_tags() -> int:
     """Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
-    images — the operator-triggered back-catalogue catch-up (task #1458 hard tier +
-    #1474 soft tier). New imports are tagged live by the importer; this covers the
-    existing library.
-
-    HARD tier ("WIP"/"work in progress") always runs (the operator triggered the
-    scan); the SOFT tier (sketch/doodle, provisional source) runs only when
-    wip_soft_title_tagging_enabled, AFTER hard so a title matching both keeps the
-    trained hard tag (ON CONFLICT DO NOTHING). Keyset-paginated, restart-safe.
+    images — the operator-triggered back-catalogue catch-up (task #1458). New
+    imports are tagged live by the importer; this covers the existing library.
+    Keyset-paginated, restart-safe.
 
     Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
     posts and silently undo a manual WIP removal, so it stays an explicit operator
     action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
     """
-    from ..models import ImportSettings
     from ..services.wip_title import (
-        SOFT_WIP_TITLE_SQL_PREFILTER,
-        WIP_TITLE_SOFT_SOURCE,
         WIP_TITLE_SOURCE,
         WIP_TITLE_SQL_PREFILTER,
-        matches_soft_wip_title,
         matches_wip_title,
         resolve_wip_tag_id,
     )
@@ -1107,16 +1222,10 @@ def backfill_wip_title_tags() -> int:
                 "backfill_wip_title_tags: no `wip` system tag present; nothing to do"
             )
             return 0
-        settings = ImportSettings.load_sync(session)
         applied = _backfill_wip_tier(
             session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
             WIP_TITLE_SOURCE,
         )
-        if settings.wip_soft_title_tagging_enabled:
-            applied += _backfill_wip_tier(
-                session, tag_id, SOFT_WIP_TITLE_SQL_PREFILTER, matches_soft_wip_title,
-                WIP_TITLE_SOFT_SOURCE,
-            )
     if applied:
         log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
     return applied

backend/app/tasks/ml.py

@@ -620,24 +620,6 @@ def scheduled_process_auto_apply() -> str:
     return f"applied={result['n_applied']} flagged={result['n_flagged']}"
 
 
-@celery.task(
-    name="backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
-    soft_time_limit=1800, time_limit=2100,
-)
-def scheduled_soft_wip_conflict_audit() -> str:
-    """Ring-loud audit over the SOFT WIP-title cohort (#1474) — flag sketch/doodle
-    auto-tags that ALSO look like real content for review. No-op when there are no
-    content heads; idempotent (already-flagged images skipped). Runs regardless of
-    the process-sweep toggle, since soft-title tags come from the importer, not that
-    sweep. Wall-clock bounded by the task time limits."""
-    from ..services.ml.heads import soft_wip_conflict_audit
-
-    SessionLocal = _sync_session_factory()
-    with SessionLocal() as session:
-        result = soft_wip_conflict_audit(session)
-    return f"scanned={result['n_scanned']} flagged={result['n_flagged']}"
-
-
 @celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
 def prune_presentation_reviews() -> str:
     """Retention (rule 89): drop RESOLVED presentation-review flags older than 30

backend/app/utils/safe_probe.py

@@ -175,12 +175,20 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
     return ("ok", None)
 
 
+def _over_cap(total) -> bool:
+    return total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES
+
+
 def _inspect_archive(path: Path):
     """Return (total_uncompressed_bytes | None, first_bad_member | None)
     for the archive. Format detected by extension OR magic bytes (so a
     mis-named archive is still bomb-guarded + integrity-tested, matching the
     extractor's gate); raises on a structurally-broken container (caught by the
-    child as a clean rejection)."""
+    child as a clean rejection).
+
+    The size total comes from the archive's directory, so an archive over the
+    cap is rejected before the integrity test decompresses anything: on a
+    multi-gigabyte archive that test alone outlasts the probe's timeout."""
     from ..services.archive_extractor import detect_archive_format
 
     fmt = detect_archive_format(path)
@@ -189,12 +197,16 @@ def _inspect_archive(path: Path):
 
         with zipfile.ZipFile(path) as zf:
             total = sum(zi.file_size for zi in zf.infolist())
+            if _over_cap(total):
+                return total, None
             return total, zf.testzip()
     if fmt == "rar":
         import rarfile
 
         with rarfile.RarFile(path) as rf:
             total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
+            if _over_cap(total):
+                return total, None
             rf.testrar()
             return total, None
     if fmt == "7z":
@@ -203,6 +215,8 @@ def _inspect_archive(path: Path):
         with py7zr.SevenZipFile(path, "r") as zf:
             info = zf.archiveinfo()
             total = getattr(info, "uncompressed", None)
+            if _over_cap(total):
+                return total, None
             ok = zf.test()  # True / None when all members pass
             return total, (None if ok in (True, None) else "7z test reported corruption")
     # Not a recognised archive — nothing to test; treat as clean.

backend/app/utils/staging.py

@@ -0,0 +1,20 @@
+"""Downloader staging files.
+
+Downloads and imports write into a staging name and `os.replace()` it into
+place when the write completes (importer, external_fetch, native_ingest_common,
+attachment_store), so a write killed midway leaves a staging file, never a
+truncated final file. Any other file is complete.
+"""
+from pathlib import Path
+
+TEMP_STAGING_SUFFIXES = (".part", ".partial")
+# megatools stages a Mega download as `.megatmp.<id>` in the target dir.
+TEMP_STAGING_PREFIXES = (".megatmp.",)
+
+
+def is_temp_staging_file(path: Path) -> bool:
+    """True for a downloader's staging file: one that never becomes library
+    content under that name, so a stale one is safe to delete."""
+    return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith(
+        TEMP_STAGING_PREFIXES
+    )

ci-requirements.md

@@ -15,7 +15,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
 
 ## Secondary runtime image
 
-node:24-bookworm-slim — `.forgejo/workflows/extension.yml` only.
+node:24-bookworm-slim — `build.yml`'s `extension-test` job only.
 
 `.forgejo/workflows/release.yml` runs on `ci-python:3.14` like everything else
 and installs nothing: it needs git and stdlib python, and builds no image.
@@ -29,8 +29,8 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
 
 - `pip install -r requirements.txt pytest pytest-asyncio` — in `backend-lint-and-test` and `integration` jobs
 - `npm install --no-audit --no-fund` — in `frontend-build` job
-- `npm install --no-audit --no-fund` — in `extension.yml`'s `lint` job (web-ext + vitest)
-- `unzip` — in `extension.yml`'s "Verify XPI contents" step, installed via apt
+- `npm install --no-audit --no-fund` — in `build.yml`'s `extension-test` job (web-ext + vitest)
+- `unzip` — in `extension-test`'s "Verify XPI contents" step, installed via apt
   only when absent (`node:24-bookworm-slim` may or may not carry it). Debian
   package, ~2s. Not worth baking into a shared image for a single consumer, per
   `docs/process.md`'s ">1 project" rule.
@@ -180,7 +180,10 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
     BUILD_REF` that every checkout in the file takes, rather than per job —
     otherwise `sign-extension` would derive dev's extension version while
     `build-web` bundled main's, and the release download would 404 on a version
-    that exists perfectly well. Every job then ASSERTS its checkout is `main`
+    that exists perfectly well. On every other trigger `BUILD_REF` is the
+    triggering COMMIT (`github.sha`), not the branch: a branch is re-resolved
+    per job, so a push landing mid-run used to move the publishing jobs onto a
+    commit the run's lanes never tested (run 7499, #4427). Every job then ASSERTS its checkout is `main`
     before doing anything, because `env` inside `with:` is not a context this
     runner is known to evaluate — if it silently resolved to empty, checkout
     would fall back to the triggering ref and the refresh would publish dev's

docker-compose.single.yml

@@ -29,7 +29,7 @@
 
 services:
   redis:
-    image: redis:7-alpine
+    image: redis:8-alpine
     volumes:
       - redis_data:/data
     healthcheck:

docker-compose.yml

@@ -48,7 +48,7 @@ x-celery-healthcheck: &celery_healthcheck
 
 services:
   redis:
-    image: redis:7-alpine
+    image: redis:8-alpine
     volumes:
       - redis_data:/data
     healthcheck: