fix: the media backup task has its own stall threshold, and its test seeds a post for a shared archive

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.forgejo/workflows/baseline.yml

@@ -81,8 +81,18 @@ jobs:
           set -eux
           # Same service-IP dance as build.yml's integration job; see the long
           # comment there for why the job name must stay separator-free.
-          PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          test -n "$PG"
+          # Only THIS job's services. The runner's docker daemon is shared, so another
+          # repo's job can be running beside this one, and a job-name filter matches its
+          # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
+          # Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
+          # <n> from our own container (its id is in the /etc/hostname bind mount) and
+          # require exactly one match.
+          SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
+          SELF=${SELF:-$(hostname)}
+          TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
+          test -n "$TASK"
+          PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
+          test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           test -n "$PG_IP"
           echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"

.forgejo/workflows/build.yml

@@ -202,7 +202,8 @@ env:
 # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
 #   - write:package, read:package  (for docker push to git.fabledsword.com)
 #   - write:release                (for ext-<version> release asset cache)
-#   - write:issue                  (for future issue-management automation)
+#   - write:issue                  (`unattended-failure` opens an issue)
+#   - write:repository             (`refresh-dev` dispatches this workflow)
 # The injected GITHUB_TOKEN cannot be used — it lacks write:package.
 
 jobs:
@@ -532,9 +533,20 @@ jobs:
           echo "=== container landscape (diagnostic for filter scoping) ==="
           docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
           echo "=== end landscape ==="
-          PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:8-alpine" -q | head -n1)
-          test -n "$PG" && test -n "$RD"
+          # Only THIS job's services. The runner's docker daemon is shared, so another
+          # repo's job can be running beside this one, and a job-name filter matches its
+          # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
+          # Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
+          # <n> from our own container (its id is in the /etc/hostname bind mount) and
+          # require exactly one match.
+          SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
+          SELF=${SELF:-$(hostname)}
+          TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
+          test -n "$TASK"
+          PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
+          test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
+          RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
+          test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
           test -n "$PG_IP" && test -n "$RD_IP"
@@ -776,6 +788,15 @@ jobs:
               print("guard: no ext-* release yet — nothing to go backwards from")
               raise SystemExit(0)
 
+          # A version that is already signed is reused from its release, not
+          # signed again, so it cannot be a downgrade. The weekly refresh builds
+          # main, whose version trails whatever dev has signed since; refusing
+          # it here would stop :latest from ever being refreshed while dev is
+          # ahead.
+          if any(v == derived for v, _ in signed):
+              print(f"guard: {derived_s} is already signed — reused, not re-signed")
+              raise SystemExit(0)
+
           hi, hi_tag = signed[-1]
           print(f"guard: derived={derived_s}  highest already signed={hi_tag}")
           if derived < hi:
@@ -1426,8 +1447,11 @@ jobs:
           # `no-cache: true`, which is a much larger version of the same
           # churn #3265 is about.
           #
-          # Only on the schedule. An ordinary push wants the cached base.
-          pull: ${{ env.IS_REFRESH == 'true' }}
+          # Only on the refresh and on a forced build. An ordinary push wants
+          # the cached base. A forced build pulls because forcing is how the
+          # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
+          # keeps the old base layers would be the same image again.
+          pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
           # ONE tag, the channel's. Every other tag is written by the step
           # below, registry-side. buildx here pushes the first tag to the
           # registry and then re-pushes the rest through the DOCKER driver,
@@ -1718,9 +1742,20 @@ jobs:
           # Service discovery mirrors the integration lane above: these jobs run
           # in a container against a mounted docker socket, so the services are
           # SIBLINGS reachable by IP, not by hostname.
-          PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:8-alpine" -q | head -n1)
-          test -n "$PG" && test -n "$RD"
+          # Only THIS job's services. The runner's docker daemon is shared, so another
+          # repo's job can be running beside this one, and a job-name filter matches its
+          # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
+          # Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
+          # <n> from our own container (its id is in the /etc/hostname bind mount) and
+          # require exactly one match.
+          SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
+          SELF=${SELF:-$(hostname)}
+          TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
+          test -n "$TASK"
+          PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
+          test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
+          RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
+          test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
           test -n "$PG_IP" && test -n "$RD_IP"
@@ -2171,6 +2206,121 @@ jobs:
             exit 1
           fi
           echo "promote: $MOVED tag(s) written"
+
+  # The refresh rebuilds `:dev` as well as `:latest`.
+  #
+  # The refresh run itself builds main and publishes `:latest`. `:dev` needs a
+  # build of dev's own source with FC_CHANNEL=dev baked in, so copying main's
+  # refreshed bytes onto `:dev` is not the same thing: the image would report
+  # itself as main, and the reuse check would then keep it that way. Instead
+  # this job dispatches an ordinary dev run with `force_build`. That run goes
+  # through the lanes, the smoke and `promote` like any push, and a forced
+  # build pulls fresh bases (see `pull:` in both build jobs).
+  #
+  # Without this, an artifact whose source is quiet on dev (the agent, most
+  # weeks) keeps the base layers of its last build indefinitely, because
+  # reuse skips its build on every push.
+  #
+  # The dispatched run shares this run's concurrency group (both are on the
+  # dev ref), so it queues until this run has finished. It runs whatever
+  # `promote` did: a failed `:latest` refresh is no reason to leave `:dev`
+  # stale, and `unattended-failure` reports the failure either way.
+  refresh-dev:
+    needs: [promote]
+    if: >-
+      !cancelled() && (github.event_name == 'schedule'
+      || format('{0}', github.event.inputs.refresh) == 'true')
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    steps:
+      - name: Dispatch a forced dev build
+        env:
+          TOKEN: ${{ secrets.RELEASE_TOKEN }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+        run: |
+          set -eu
+          CODE=$(curl -sS --max-time 30 -o response.txt -w '%{http_code}' -X POST \
+            -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
+            -d '{"ref":"dev","inputs":{"force_build":"true"}}' \
+            "$API/actions/workflows/build.yml/dispatches")
+          echo "refresh-dev: dispatch answered HTTP $CODE"
+          case "$CODE" in
+            2??) echo "refresh-dev: a forced dev run is queued behind this one" ;;
+            *)
+              cat response.txt >&2
+              echo "refresh-dev: the dispatch was refused. A 403 means RELEASE_TOKEN" >&2
+              echo "refresh-dev: lacks write:repository (see the scope list at the top)." >&2
+              exit 1 ;;
+          esac
+
+  # A run nobody is watching reports its own failure.
+  #
+  # A push run is watched by whoever pushed it. The weekly refresh and the dev
+  # run it dispatches are not, and a red run on the Actions page alone goes
+  # unseen. So any failed run that is not a push or a pull request opens an
+  # issue, or comments on the one already open, naming the run and the jobs
+  # that failed. A manual dispatch that fails reports here too; closing the
+  # issue once it is dealt with is what keeps the next failure visible.
+  #
+  # The issue is written with RELEASE_TOKEN, so it is authored by the token's
+  # owner, and the forge does not notify people of their own actions. It
+  # shows in the repository's open issues rather than as an email.
+  unattended-failure:
+    needs: [lint, extension-version, backend-lint-and-test, frontend-build,
+            extension-test, integration, sign-extension, build-web, smoke-web,
+            build-agent, promote, refresh-dev]
+    if: >-
+      failure() && github.event_name != 'push'
+      && github.event_name != 'pull_request'
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    steps:
+      - name: Open or update the failure issue
+        env:
+          TOKEN: ${{ secrets.RELEASE_TOKEN }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          EVENT: ${{ github.event_name }}
+          REF: ${{ github.ref_name }}
+          NEEDS: ${{ toJSON(needs) }}
+        run: |
+          python3 - <<'EOF'
+          import json, os, urllib.parse, urllib.request
+
+          TITLE = "CI: an unattended build.yml run failed"
+          api, token = os.environ["API"], os.environ["TOKEN"]
+
+          def call(method, path, body=None):
+              req = urllib.request.Request(
+                  api + path, method=method,
+                  data=json.dumps(body).encode() if body is not None else None,
+                  headers={"Authorization": f"token {token}",
+                           "Content-Type": "application/json"})
+              with urllib.request.urlopen(req, timeout=30) as resp:
+                  return json.load(resp)
+
+          needs = json.loads(os.environ["NEEDS"])
+          failed = sorted(j for j, v in needs.items() if v.get("result") == "failure")
+          text = (f"Run: {os.environ['RUN_URL']}\n"
+                  f"Trigger: `{os.environ['EVENT']}` on `{os.environ['REF']}`\n"
+                  f"Failed jobs: {', '.join(f'`{j}`' for j in failed) or '(none reported)'}\n")
+
+          q = urllib.parse.urlencode({"state": "open", "type": "issues", "q": TITLE})
+          existing = [i for i in call("GET", f"/issues?{q}") if i["title"] == TITLE]
+          if existing:
+              n = existing[0]["number"]
+              call("POST", f"/issues/{n}/comments", {"body": text})
+              print(f"unattended-failure: commented on open issue #{n}")
+          else:
+              body = ("A build.yml run that nobody watches failed. Read the run's "
+                      "failing job logs; close this issue once it is fixed, so the "
+                      "next failure opens a new one.\n\n" + text)
+              n = call("POST", "/issues", {"title": TITLE, "body": body})["number"]
+              print(f"unattended-failure: opened issue #{n}")
+          EOF
+
   build-agent:
     # THE GATE (2026-09-23). Every lane above must have PASSED before this job
     # exists at all — so a red suite does not produce an image, let alone push
@@ -2509,8 +2659,11 @@ jobs:
           # `no-cache: true`, which is a much larger version of the same
           # churn #3265 is about.
           #
-          # Only on the schedule. An ordinary push wants the cached base.
-          pull: ${{ env.IS_REFRESH == 'true' }}
+          # Only on the refresh and on a forced build. An ordinary push wants
+          # the cached base. A forced build pulls because forcing is how the
+          # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
+          # keeps the old base layers would be the same image again.
+          pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
           # ONE tag, the channel's. Every other tag is written by the step
           # below, registry-side. buildx here pushes the first tag to the
           # registry and then re-pushes the rest through the DOCKER driver,

alembic/versions/0115_source_display_name.py

@@ -0,0 +1,27 @@
+"""source.display_name — what a source is called on its platform.
+
+#4481. A Discord source's URL is a server id and a channel id, so the
+Subscriptions page could only show two numbers. The Discord ingester already
+loads the server and channel names to walk them; it now keeps them here,
+refreshed on every walk. NULL until a walk has read it.
+
+Revision ID: 0115
+Revises: 0114
+Create Date: 2026-09-28
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0115"
+down_revision = "0114"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("source", "display_name")

alembic/versions/0116_single_color_threshold.py

@@ -0,0 +1,34 @@
+"""The single-color filter's default becomes near-total: 0.95 → 0.995.
+
+#4483. At 0.95 the filter rejected line art: a pencil doodle on white is
+mostly white, and five of Todding's Discord doodles were skipped on import as
+"single color", leaving their posts with text and no image. The predicate now
+samples real pixels instead of a blurred thumbnail, and the default only
+calls an image blank when it essentially is one.
+
+A stored 0.95 is the old default, so it moves with it. Any other value was
+chosen by hand and is left alone.
+
+Revision ID: 0116
+Revises: 0115
+Create Date: 2026-09-28
+
+"""
+from alembic import op
+
+revision = "0116"
+down_revision = "0115"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.alter_column("import_settings", "single_color_threshold", server_default="0.995")
+    op.execute(
+        "UPDATE import_settings SET single_color_threshold = 0.995 "
+        "WHERE single_color_threshold = 0.95"
+    )
+
+
+def downgrade() -> None:
+    op.alter_column("import_settings", "single_color_threshold", server_default="0.95")

alembic/versions/0117_external_link_started_at.py

@@ -0,0 +1,33 @@
+"""external_link.started_at: when a fetch claimed the link.
+
+A worker killed mid-fetch (the Celery hard limit, an OOM, a restart) never
+reaches the code that moves its link out of 'downloading'. The external-link
+sweep returns such a link to the retry path once it has been 'downloading'
+longer than the task's hard limit, which needs the claim time.
+
+A row in 'downloading' with no claim time is treated as stale: no worker
+survives the deploy that adds this column.
+
+Revision ID: 0117
+Revises: 0116
+Create Date: 2026-11-05
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0117"
+down_revision = "0116"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "external_link",
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("external_link", "started_at")

alembic/versions/0118_external_link_recoveries.py

@@ -0,0 +1,32 @@
+"""external_link.recoveries: how many times recovery has retried a dead link.
+
+The daily recovery gives a dead link one more attempt, spaced further apart
+each time, and leaves it dead once the count reaches its limit. A link that
+can never succeed is then downloaded a handful of times in total, not every
+day.
+
+Revision ID: 0118
+Revises: 0117
+Create Date: 2026-11-05
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0118"
+down_revision = "0117"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "external_link",
+        sa.Column(
+            "recoveries", sa.Integer(), nullable=False, server_default=sa.text("0")
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("external_link", "recoveries")

alembic/versions/0119_backup_s3_destination.py

@@ -0,0 +1,43 @@
+"""import_settings: the off-site backup destination (an S3 bucket).
+
+Address, bucket, access key and region are plain settings; the secret key is
+stored sealed with the credential key. All empty means no destination is set
+up, which is every existing install's state.
+
+Revision ID: 0119
+Revises: 0118
+Create Date: 2026-11-06
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0119"
+down_revision = "0118"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("import_settings", sa.Column("backup_s3_endpoint", sa.Text(), nullable=True))
+    op.add_column(
+        "import_settings", sa.Column("backup_s3_bucket", sa.String(255), nullable=True),
+    )
+    op.add_column("import_settings", sa.Column("backup_s3_access_key", sa.Text(), nullable=True))
+    op.add_column(
+        "import_settings", sa.Column("backup_s3_secret_sealed", sa.Text(), nullable=True),
+    )
+    op.add_column(
+        "import_settings",
+        sa.Column(
+            "backup_s3_region", sa.String(64), nullable=False, server_default="us-east-1",
+        ),
+    )
+
+
+def downgrade() -> None:
+    for column in (
+        "backup_s3_region", "backup_s3_secret_sealed", "backup_s3_access_key",
+        "backup_s3_bucket", "backup_s3_endpoint",
+    ):
+        op.drop_column("import_settings", column)

alembic/versions/0120_backup_upload_limit.py

@@ -0,0 +1,30 @@
+"""import_settings: a bandwidth limit for uploads to the off-site backup bucket.
+
+In megabits per second; 0 means no limit. Every install starts at 200 Mbit/s,
+which leaves room on a gigabit link for everything else the server does.
+
+Revision ID: 0120
+Revises: 0119
+Create Date: 2026-11-07
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0120"
+down_revision = "0119"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "import_settings",
+        sa.Column(
+            "backup_upload_limit_mbps", sa.Integer(), nullable=False, server_default="200",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("import_settings", "backup_upload_limit_mbps")

alembic/versions/0121_backup_blob.py

@@ -0,0 +1,46 @@
+"""backup_blob: which library files are already in the off-site bucket.
+
+One row per (destination, sha256) sent. The media backup's work list is every
+image and attachment sha with no row for the current destination, so a run is
+incremental and resumable by construction, and pointing backups at a new
+bucket starts that bucket's copy from the beginning.
+
+backup_run.last_progress_at is the heartbeat of a run that spans hours; the
+stall sweep measures a media run's silence from it, not from its start.
+
+Revision ID: 0121
+Revises: 0120
+Create Date: 2026-11-07
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0121"
+down_revision = "0120"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "backup_blob",
+        sa.Column("target", sa.Text(), nullable=False),
+        sa.Column("sha256", sa.String(64), nullable=False),
+        sa.Column("key", sa.Text(), nullable=False),
+        sa.Column("size_bytes", sa.BigInteger(), nullable=False),
+        sa.Column(
+            "uploaded_at", sa.DateTime(timezone=True), nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.PrimaryKeyConstraint("target", "sha256", name=op.f("pk_backup_blob")),
+    )
+    op.add_column(
+        "backup_run",
+        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("backup_run", "last_progress_at")
+    op.drop_table("backup_blob")

backend/app/api/admin.py

@@ -475,6 +475,21 @@ async def trigger_reclaim_attachments():
     return _queued(async_result)
 
 
+@admin_bp.route("/maintenance/import-unimported", methods=["POST"])
+async def trigger_import_unimported():
+    """Import files in the artists' folders that the library has no record of.
+    Body {"dry_run": bool}; dry_run (the DEFAULT) lists what was found, per
+    artist, and changes nothing. dry_run=false queues them as an import batch
+    that attaches each file where it lies. Returns the Celery task id — poll
+    /maintenance/task-result/<id> for the summary."""
+    from ..tasks.admin import import_unimported_files_task
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", True))
+    async_result = import_unimported_files_task.delay(dry_run=dry_run)
+    return _queued(async_result)
+
+
 @admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
 async def trigger_repair_discord_downloads():
     """Clean re-download of the Discord files broken by the `None` naming

backend/app/api/extension.py

@@ -19,8 +19,10 @@ from ..models import AppSetting
 from ..services.extension_service import (
     ExtensionService,
     InvalidUrlError,
+    PosterLookupError,
     UnknownArtistError,
     UnknownPlatformError,
+    UnknownSourceError,
 )
 from ..services.source_service import KNOWN_PLATFORMS
 from ._responses import error_response as _bad
@@ -121,6 +123,10 @@ async def quick_add_source():
     # Patreon is canon: adding a Patreon source to an existing artist can take
     # the creator's Patreon display name (name only; the slug never moves).
     use_platform_name = body.get("use_platform_name") is True
+    # #4491: a Discord add can carry the posters picked in the panel.
+    discord_authors = body.get("discord_authors")
+    if discord_authors is not None and not _valid_authors(discord_authors):
+        return _bad("invalid_body", detail="discord_authors must be a list of {id, name}")
 
     from .credentials import _get_crypto
 
@@ -133,6 +139,7 @@ async def quick_add_source():
             result = await ExtensionService(session, _get_crypto()).quick_add_source(
                 url, artist_id=artist_id, artist_name=artist_name,
                 use_platform_name=use_platform_name,
+                discord_authors=discord_authors,
             )
         except UnknownArtistError as exc:
             return _bad("not_found", detail=str(exc), status=404)
@@ -147,6 +154,59 @@ async def quick_add_source():
     return jsonify(result), (201 if result["created_source"] else 200)
 
 
+def _valid_authors(value) -> bool:
+    """[{id, name}]: the id is what the list keys on, the name only shows it."""
+    return isinstance(value, list) and all(
+        isinstance(a, dict) and isinstance(a.get("id"), str | int)
+        and not isinstance(a.get("id"), bool)
+        and (a.get("name") is None or isinstance(a.get("name"), str))
+        for a in value
+    )
+
+
+@extension_bp.route("/discord/posters", methods=["GET"])
+async def discord_posters():
+    """Who posts in the Discord channel `url` shows, lately, with the likely
+    creator marked (#4491). Reads the channel with the stored token; writes
+    nothing."""
+    url = (request.args.get("url") or "").strip()
+    if not url:
+        return _bad("invalid_body", detail="url query parameter is required")
+    from .credentials import _get_crypto
+
+    async with get_session() as session:
+        if not await _ext_key_required(session):
+            return _bad("unauthorized", status=401)
+        try:
+            result = await ExtensionService(session, _get_crypto()).discord_posters(url)
+        except (InvalidUrlError, UnknownPlatformError) as exc:
+            return _bad("invalid_url", detail=str(exc))
+        except PosterLookupError as exc:
+            return _bad("posters_unavailable", detail=str(exc), status=409)
+    return jsonify(result)
+
+
+@extension_bp.route("/discord/posters", methods=["POST"])
+async def set_discord_posters():
+    """Set the poster list of the source following `url`'s channel."""
+    body = await request.get_json(silent=True)
+    if not isinstance(body, dict) or not isinstance(body.get("url"), str):
+        return _bad("invalid_body", detail="url is required")
+    authors = body.get("authors")
+    if not _valid_authors(authors):
+        return _bad("invalid_body", detail="authors must be a list of {id, name}")
+    async with get_session() as session:
+        if not await _ext_key_required(session):
+            return _bad("unauthorized", status=401)
+        try:
+            result = await ExtensionService(session).set_discord_posters(body["url"], authors)
+        except (InvalidUrlError, UnknownPlatformError, PosterLookupError) as exc:
+            return _bad("invalid_url", detail=str(exc))
+        except UnknownSourceError as exc:
+            return _bad("not_found", detail=str(exc), status=404)
+    return jsonify(result)
+
+
 def _read_manifest_sync() -> dict | None:
     """All the filesystem-touching work for /api/extension/manifest,
     in a sync helper so the async route can dispatch it via

backend/app/api/sources.py

@@ -1,10 +1,13 @@
 """FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
 
+from pathlib import Path
+
 from quart import Blueprint, jsonify, request
 from sqlalchemy import func, select
 
 from ..extensions import get_session
 from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
+from ..services import discord_poster_cleanup as poster_cleanup
 from ..services.artist_membership_service import ArtistMembershipService
 from ..services.artist_membership_service import rescan as membership_rescan
 from ..services.artist_service import ArtistService
@@ -463,3 +466,49 @@ async def adopt_membership():
             return jsonify({"already_tracked": exc.existing_id})
         artist_id = artist.id
     return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
+
+
+# -- Discord: remove posts by people outside the poster list (#4486) --------------
+
+_IMAGES_ROOT = Path("/images")
+
+
+async def _poster_cleanup_preview(source_id: int):
+    async with get_session() as session:
+        return await session.run_sync(
+            lambda s: poster_cleanup.preview(s, source_id=source_id)
+        )
+
+
+@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
+async def other_posters_preview(source_id: int):
+    """What removing other posters' posts would delete. Nothing is touched."""
+    try:
+        projection = await _poster_cleanup_preview(source_id)
+    except LookupError:
+        return _bad("not_found", status=404)
+    except poster_cleanup.PosterCleanupError as exc:
+        return _bad("not_applicable", detail=str(exc), status=409)
+    projection["confirm_token"] = poster_cleanup.confirm_token(projection)
+    return jsonify(projection)
+
+
+@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
+async def other_posters_remove(source_id: int):
+    """Delete them. `confirm` must be the token of the preview the operator saw:
+    a poster list edited since then changes the set, and the token with it."""
+    body = await request.get_json(silent=True) or {}
+    try:
+        projection = await _poster_cleanup_preview(source_id)
+    except LookupError:
+        return _bad("not_found", status=404)
+    except poster_cleanup.PosterCleanupError as exc:
+        return _bad("not_applicable", detail=str(exc), status=409)
+    expected = poster_cleanup.confirm_token(projection)
+    if body.get("confirm") != expected:
+        return _bad("confirm_mismatch", expected=expected)
+    async with get_session() as session:
+        result = await session.run_sync(
+            lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
+        )
+    return jsonify(result)

backend/app/api/system_backup.py

@@ -20,7 +20,7 @@ system_backup_bp = Blueprint(
     "system_backup", __name__, url_prefix="/api/system/backup",
 )
 
-_KINDS = frozenset({"db", "images"})
+_KINDS = frozenset({"db", "images", "media"})
 _TAG_MAX_LEN = 64
 _BACKUP_SETTINGS_FIELDS = (
     "backup_db_nightly_enabled",
@@ -49,6 +49,9 @@ def _row_to_dict(r: BackupRun) -> dict:
         "error": r.error,
         "restored_from_id": r.restored_from_id,
         "manifest": r.manifest or {},
+        # Where an off-site dump went ({endpoint, bucket, key}); None for a
+        # backup kept on this server.
+        "offsite": (r.manifest or {}).get("offsite"),
     }
 
 
@@ -107,6 +110,38 @@ async def trigger_images_backup():
     return jsonify({"status": "dispatched"}), 202
 
 
+@system_backup_bp.route("/media", methods=["POST"])
+async def trigger_media_backup():
+    """Send the images and attachments not yet in the off-site bucket. Refused
+    when no destination is set up; a run already going is left to finish."""
+    from ..services import backup_destination as dest
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        if not dest.describe(row)["configured"]:
+            return _bad("no_destination",
+                        detail="set up an off-site destination before sending files")
+    from ..tasks.backup import backup_media_task
+    backup_media_task.delay(triggered_by="manual")
+    return jsonify({"status": "dispatched"}), 202
+
+
+@system_backup_bp.route("/media", methods=["GET"])
+async def media_coverage():
+    """How much of the library is in the off-site bucket: files and bytes,
+    sent and total, each file counted once however many posts hold it."""
+    from ..services import backup_destination as dest
+    from ..services import backup_media
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        f = dest.fields_of(row)
+        configured = dest.describe(row)["configured"]
+        tid = backup_media.destination_id(f.endpoint, f.bucket) if configured else None
+        counts = await session.run_sync(lambda s: backup_media.coverage(s, tid))
+    return jsonify({"configured": configured, **counts})
+
+
 @system_backup_bp.route("/runs", methods=["GET"])
 async def list_runs():
     try:
@@ -180,6 +215,12 @@ async def trigger_restore(run_id: int):
                 "not_restorable",
                 detail=f"source backup status={row.status!r}; only 'ok' rows are restorable",
             )
+        if (row.manifest or {}).get("offsite"):
+            return _bad(
+                "not_restorable",
+                detail=("this dump is kept in the off-site bucket, not on this server, "
+                        "so it can't be restored from here"),
+            )
         expected = f"restore-{row.kind}-{row.id}"
         if supplied != expected:
             return _bad(
@@ -236,6 +277,118 @@ async def get_settings():
     })
 
 
+@system_backup_bp.route("/destination", methods=["GET"])
+async def get_destination():
+    """The off-site destination as the settings screen shows it. The secret
+    key is never returned, only whether one is saved."""
+    from ..services import backup_destination as dest
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        return jsonify(dest.describe(row))
+
+
+@system_backup_bp.route("/destination", methods=["PATCH"])
+async def patch_destination():
+    """Save the destination. Body {endpoint, bucket, access_key, region,
+    secret_key, upload_limit_mbps}; an empty field clears it, and a missing
+    upload_limit_mbps keeps the saved limit. A missing or empty secret_key
+    keeps the saved one, so the screen can save the other fields without the
+    secret being typed again."""
+    from ..services import backup_destination as dest
+
+    body = await request.get_json(silent=True)
+    if not isinstance(body, dict):
+        return _bad("invalid_body", detail="body must be a JSON object")
+    fields, why = dest.clean_fields(body)
+    if fields is None:
+        return _bad("invalid_destination", detail=why)
+    secret = body.get("secret_key")
+    if secret is not None and not isinstance(secret, str):
+        return _bad("invalid_destination", detail="secret_key must be text")
+    limit = body.get("upload_limit_mbps")
+    if limit is not None and (
+        not isinstance(limit, int) or isinstance(limit, bool) or not 0 <= limit <= 100_000
+    ):
+        return _bad("invalid_destination",
+                    detail="upload_limit_mbps must be a whole number from 0 (no limit) up")
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        if limit is not None:
+            row.backup_upload_limit_mbps = limit
+        row.backup_s3_endpoint = fields.endpoint
+        row.backup_s3_bucket = fields.bucket
+        row.backup_s3_access_key = fields.access_key
+        row.backup_s3_region = fields.region
+        if secret and secret.strip():
+            row.backup_s3_secret_sealed = dest.seal(secret.strip())
+        await session.commit()
+        return jsonify(dest.describe(row))
+
+
+@system_backup_bp.route("/destination", methods=["DELETE"])
+async def delete_destination():
+    """Forget the destination, secret included. What was already sent stays in
+    the bucket: FabledCurator never deletes there."""
+    from ..services import backup_destination as dest
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        row.backup_s3_endpoint = None
+        row.backup_s3_bucket = None
+        row.backup_s3_access_key = None
+        row.backup_s3_secret_sealed = None
+        row.backup_s3_region = dest.DEFAULT_REGION
+        await session.commit()
+        return jsonify(dest.describe(row))
+
+
+@system_backup_bp.route("/destination/test", methods=["POST"])
+async def try_destination():
+    """Reach the bucket with the fields in the body (the form as typed, saved
+    or not); a missing secret_key uses the saved one. Lists the bucket and
+    writes nothing. Answers {ok, message} either way: a destination that
+    can't be reached is a result to show, not an API error."""
+    import asyncio
+
+    from ..services import backup_destination as dest
+    from ..services.backup_s3 import S3Error
+    from ..services.credential_crypto import InvalidCredentialBlob
+
+    body = await request.get_json(silent=True) or {}
+    if not isinstance(body, dict):
+        return _bad("invalid_body", detail="body must be a JSON object")
+    fields, why = dest.clean_fields(body)
+    if fields is None:
+        return jsonify({"ok": False, "message": f"Not tested: {why}."})
+    secret = body.get("secret_key")
+    secret = secret.strip() if isinstance(secret, str) else ""
+    if not secret:
+        async with get_session() as session:
+            row = await ImportSettings.load(session)
+            sealed = row.backup_s3_secret_sealed
+        if sealed:
+            try:
+                secret = dest.unseal(sealed)
+            except InvalidCredentialBlob:
+                return jsonify({"ok": False, "message": (
+                    "Not tested: the saved secret key can't be opened on this server. "
+                    "Enter the secret key again."
+                )})
+    target = dest.target_for(fields, secret or None)
+    if target is None:
+        missing = [name for name, v in (
+            ("address", fields.endpoint), ("bucket", fields.bucket),
+            ("access key", fields.access_key), ("secret key", secret),
+        ) if not v]
+        return jsonify({"ok": False, "message": f"Not tested: fill in the {', '.join(missing)}."})
+    try:
+        return jsonify(await asyncio.to_thread(dest.check_target, target))
+    except S3Error as exc:
+        return jsonify({"ok": False, "message": str(exc)})
+
+
 @system_backup_bp.route("/settings", methods=["PATCH"])
 async def patch_settings():
     body = await request.get_json(silent=True)

backend/app/celery_app.py

@@ -302,6 +302,10 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.backup.prune_backups",
                 "schedule": 86400.0,  # daily
             },
+            "reconcile-media-backup": {
+                "task": "backend.app.tasks.backup.reconcile_media_backup",
+                "schedule": 604800.0,  # weekly: list the bucket, re-send what's missing
+            },
             # Audit 2026-06-02 — three new per-entity recovery sweeps.
             # Each runs every 5 min like the other recover_stalled_*
             # sweeps; each is a no-op when nothing is stuck.

backend/app/models/__init__.py

@@ -4,6 +4,7 @@ from .app_setting import AppSetting
 from .artist import Artist
 from .artist_membership_suggestion import ArtistMembershipSuggestion
 from .artist_visit import ArtistVisit
+from .backup_blob import BackupBlob
 from .backup_run import BackupRun
 from .base import Base
 from .character_prototype import CcipPrototypeState, CharacterPrototype
@@ -55,6 +56,7 @@ __all__ = [
     "Artist",
     "ArtistMembershipSuggestion",
     "ArtistVisit",
+    "BackupBlob",
     "BackupRun",
     "Source",
     "Credential",

backend/app/models/backup_blob.py

@@ -0,0 +1,26 @@
+"""backup_blob: a library file already sent to the off-site bucket.
+
+One row per (destination, sha256). `target` names the destination as
+"<endpoint>/<bucket>", so pointing backups at another bucket leaves these rows
+inert and that bucket's copy starts from the beginning. `key` is where the
+file sits in the bucket (`media/ab/cd/<sha256><ext>`).
+"""
+
+from datetime import datetime
+
+from sqlalchemy import BigInteger, DateTime, String, Text, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class BackupBlob(Base):
+    __tablename__ = "backup_blob"
+
+    target: Mapped[str] = mapped_column(Text, primary_key=True)
+    sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
+    key: Mapped[str] = mapped_column(Text, nullable=False)
+    size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
+    uploaded_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )

backend/app/models/backup_run.py

@@ -1,6 +1,7 @@
 """FC-3h: backup_run — operator-facing artifact record for a backup run.
 
-One row per backup attempt (kind='db' or 'images'). Lifecycle
+One row per backup attempt (kind='db', 'images', or 'media' for the off-site
+upload of images and attachments, which spans many task chunks). Lifecycle
 tracking (started_at/finished_at/duration_ms/exception text) lives
 in task_run from FC-3i — this row records artifact metadata: file
 paths, sizes, tag (retention protection), and restore lineage via
@@ -62,6 +63,11 @@ class BackupRun(Base):
     )
     # Self-referential FK, unindexed until 0089 (#3300): SET NULL has to find
     # the rows pointing at a deleted run before it can null them.
+    # Heartbeat of a run that spans hours (the media backup): set as it makes
+    # progress, and what the stall sweep measures its silence from.
+    last_progress_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )
     restored_from_id: Mapped[int | None] = mapped_column(
         ForeignKey("backup_run.id", ondelete="SET NULL"),
         nullable=True, index=True,

backend/app/models/external_link.py

@@ -80,12 +80,23 @@ class ExternalLink(Base):
         Integer, nullable=False, server_default=text("0")
     )
     last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Times the daily recovery has given this link another attempt after it
+    # went dead. Recovery stops at a limit, so a link that can never succeed
+    # isn't downloaded forever.
+    recoveries: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
     attachment_id: Mapped[int | None] = mapped_column(
         ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
     )
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
+    # When the current fetch claimed the link ('downloading'). The sweep uses it
+    # to recover a link whose worker died mid-fetch.
+    started_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
     completed_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )

backend/app/models/import_settings.py

@@ -9,6 +9,7 @@ from sqlalchemy import (
     CheckConstraint,
     Float,
     Integer,
+    String,
     Text,
     select,
     text,
@@ -39,7 +40,7 @@ class ImportSettings(Base):
     transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
 
     skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
-    single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
+    single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995")
     single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
 
     # Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The
@@ -100,6 +101,20 @@ class ImportSettings(Base):
         Integer, nullable=False, default=3,
         server_default="3",
     )
+    # Off-site backup destination: an S3 bucket (services/backup_destination).
+    # The secret key is stored sealed with the credential key, never in clear.
+    backup_s3_endpoint: Mapped[str | None] = mapped_column(Text, nullable=True)
+    backup_s3_bucket: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    backup_s3_access_key: Mapped[str | None] = mapped_column(Text, nullable=True)
+    backup_s3_secret_sealed: Mapped[str | None] = mapped_column(Text, nullable=True)
+    backup_s3_region: Mapped[str] = mapped_column(
+        String(64), nullable=False, default="us-east-1",
+        server_default="us-east-1",
+    )
+    # Upload bandwidth to that bucket, in megabits per second; 0 = no limit.
+    backup_upload_limit_mbps: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=200, server_default="200",
+    )
 
     # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
     # the weighted-score cut-off (0..1) above which a pending suggestion is made.

backend/app/models/source.py

@@ -46,6 +46,11 @@ class Source(Base):
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
 
     config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+    # alembic 0115: what the source is called on its platform, where the URL
+    # alone doesn't say — a Discord link is two numbers. The ingester that
+    # walks it refreshes it every walk, so a rename reads as the new name.
+    # NULL until a walk has read it, and on platforms that don't write it.
+    display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
 
     last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     last_error: Mapped[str | None] = mapped_column(Text, nullable=True)

backend/app/services/audits/single_color.py

@@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
 2026-07-02 — the import-side filter (Importer._single_color_hit /
 SkipReason.single_color), so what the audit flags and what the import
 skips can never disagree.
+
+It is meant to catch the blank: a placeholder, an error tile, a solid fill.
+Line art is the case it must not catch (#4483): a pencil doodle on white can be
+well over 95% white even at full size, and a smoothing downsample blends its strokes
+into the paper until it measures as blank. So the sample is taken by NEAREST
+(each sampled pixel is a real pixel, strokes keep their contrast), at 256px
+so thin strokes are still hit, and the default threshold is near-total
+(0.995) — a few percent of ink is a drawing, not an empty image.
 """
 
 from PIL import Image
 
-_THUMB_SIZE = (64, 64)
+_THUMB_SIZE = (256, 256)
 
 
 def evaluate(
@@ -20,7 +28,8 @@ def evaluate(
     """True iff the fraction of pixels within `tolerance` (Euclidean RGB
     distance) of the dominant color exceeds `threshold`.
 
-    Downsamples to 64x64 for speed (~4ms regardless of source size).
+    Samples 256x256 by NEAREST (see the module docstring for why not a
+    smoothing resample).
     Alpha channels are stripped; only RGB is considered. Animated images
     use frame 0 (PIL's default after Image.open without seek).
     """
@@ -30,7 +39,7 @@ def evaluate(
     elif im.mode not in ("RGB", "L"):
         im = im.convert("RGB")
     if im.size != _THUMB_SIZE:
-        im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
+        im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST)
     pixels = list(im.getdata())
     if not pixels:
         return False

backend/app/services/backup_destination.py

@@ -0,0 +1,138 @@
+"""The off-site backup destination: an S3 bucket (Fabled Archives, or any S3
+service) that the database dumps and the media blobs are sent to.
+
+The address, bucket, access key and region are plain settings. The secret key
+is sealed at rest with the credential key (`/images/secrets`), the same key
+that seals account logins, and no API ever returns it: the settings answer
+only whether one is set. Because the credential key is kept out of every
+backup, a database dump carries the secret only as ciphertext nobody can open
+without the original server.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from .backup_s3 import S3Error, S3Target, list_objects
+from .credential_crypto import CredentialCrypto, InvalidCredentialBlob
+
+KEY_PATH = Path("/images/secrets/credential_key.b64")
+DEFAULT_REGION = "us-east-1"
+_TEST_LISTING = 5
+
+
+def _crypto() -> CredentialCrypto:
+    return CredentialCrypto(KEY_PATH)
+
+
+def seal(secret: str) -> str:
+    return _crypto().encrypt(secret).decode("ascii")
+
+
+def unseal(sealed: str) -> str:
+    return _crypto().decrypt(sealed.encode("ascii"))
+
+
+@dataclass(frozen=True)
+class DestinationFields:
+    endpoint: str | None
+    bucket: str | None
+    access_key: str | None
+    region: str
+
+
+def clean_fields(body: dict) -> tuple[DestinationFields | None, str | None]:
+    """The destination fields from a request body, trimmed, with empty ones as
+    None; or (None, why) when the address can't be an S3 endpoint."""
+    def text(name: str) -> str | None:
+        v = body.get(name)
+        if v is None:
+            return None
+        if not isinstance(v, str):
+            raise TypeError(name)
+        return v.strip() or None
+
+    try:
+        endpoint = text("endpoint")
+        bucket = text("bucket")
+        access_key = text("access_key")
+        region = text("region") or DEFAULT_REGION
+    except TypeError as exc:
+        return None, f"{exc.args[0]} must be text"
+    if endpoint is not None:
+        u = urlsplit(endpoint)
+        if u.scheme not in ("http", "https") or not u.hostname:
+            return None, ("the address must start with http:// or https:// and name "
+                          "a host, e.g. http://archives.example:8081")
+        if u.query or u.fragment:
+            return None, "the address can't carry a query or a fragment"
+        endpoint = endpoint.rstrip("/")
+    if bucket is not None and "/" in bucket:
+        return None, "the bucket is a name, not a path: put any path in the address"
+    return DestinationFields(endpoint, bucket, access_key, region), None
+
+
+def target_for(fields: DestinationFields, secret: str | None) -> S3Target | None:
+    """The S3 target, or None while any part of it is missing."""
+    if not (fields.endpoint and fields.bucket and fields.access_key and secret):
+        return None
+    return S3Target(fields.endpoint, fields.bucket, fields.access_key, secret,
+                    region=fields.region)
+
+
+def saved_target(settings) -> S3Target | None:
+    """The destination saved in ImportSettings, or None when none is set up.
+    Raises S3Error when the secret can't be opened (the credential key
+    changed since it was saved)."""
+    sealed = settings.backup_s3_secret_sealed
+    if not sealed:
+        return None
+    try:
+        secret = unseal(sealed)
+    except InvalidCredentialBlob:
+        raise S3Error(
+            "The saved secret key can't be opened on this server (the credential "
+            "key changed since it was saved). Enter the secret key again."
+        ) from None
+    return target_for(fields_of(settings), secret)
+
+
+def fields_of(settings) -> DestinationFields:
+    return DestinationFields(
+        settings.backup_s3_endpoint, settings.backup_s3_bucket,
+        settings.backup_s3_access_key, settings.backup_s3_region or DEFAULT_REGION,
+    )
+
+
+def describe(settings) -> dict:
+    """The destination as the settings screen shows it: never the secret."""
+    f = fields_of(settings)
+    return {
+        "endpoint": f.endpoint or "",
+        "bucket": f.bucket or "",
+        "access_key": f.access_key or "",
+        "region": f.region,
+        "secret_set": bool(settings.backup_s3_secret_sealed),
+        "upload_limit_mbps": settings.backup_upload_limit_mbps,
+        "configured": bool(
+            f.endpoint and f.bucket and f.access_key and settings.backup_s3_secret_sealed
+        ),
+    }
+
+
+def check_target(target: S3Target) -> dict:
+    """Reach the bucket by listing it. A listing proves the address, the
+    bucket and both keys at once, and writes nothing, because an Archives key
+    is allowed to add objects but a test object could never be removed."""
+    try:
+        page = list_objects(target, limit=_TEST_LISTING)
+    except S3Error as exc:
+        return {"ok": False, "message": str(exc)}
+    if not page.objects:
+        said = f"Reached {target.bucket} at {target.endpoint}. The bucket is empty."
+    else:
+        more = "+" if page.next_token else ""
+        said = (f"Reached {target.bucket} at {target.endpoint}. It holds "
+                f"{len(page.objects)}{more} object(s), e.g. {page.objects[0].key}.")
+    return {"ok": True, "message": said}

backend/app/services/backup_media.py

@@ -0,0 +1,202 @@
+"""The off-site copy of the library's files: every image and attachment, sent
+once each to the backup bucket as a content-addressed object.
+
+Layout in the bucket, beside the database dumps under `database/`:
+
+    media/ab/cd/<sha256><ext>
+
+`ab` and `cd` are the first four hex digits of the file's sha256, which keeps
+any one prefix small, and the extension keeps the files readable in the bucket
+without FabledCurator. The sha256 is the one recorded at import, so a file
+uploads exactly once however many rows or posts point at it, and a run is
+incremental: its work list is every sha with no `backup_blob` row for the
+current destination. A file removed from the library stays in the bucket until
+the bucket's own retention removes it; FabledCurator never deletes there.
+
+The bucket checks every object against that recorded sha256, so a file whose
+bytes changed on disk since import is refused and reported, never stored under
+the wrong name.
+"""
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import delete, select, text
+from sqlalchemy.dialects.postgresql import insert
+
+from ..models import BackupBlob
+from .backup_s3 import (
+    Pacer,
+    S3Error,
+    S3Target,
+    list_objects,
+    object_present,
+    put_file,
+    put_file_in_parts,
+)
+
+PREFIX = "media/"
+# Files above this size go in parts (S3 refuses a single PUT over 5 GiB, and a
+# part that fails is resent alone instead of the whole file).
+MULTIPART_OVER = 1 << 30
+# The bucket is listed this many keys at a time during a reconcile, with a
+# short rest between pages.
+LIST_PAGE = 1000
+LIST_REST_SECONDS = 0.2
+_EXT = re.compile(r"^\.[a-z0-9]{1,10}$")
+
+
+@dataclass(frozen=True)
+class Blob:
+    sha256: str
+    path: str
+    size_bytes: int
+    mime: str | None
+
+
+def destination_id(endpoint: str, bucket: str) -> str:
+    """How backup_blob names a destination: one copy per endpoint and bucket."""
+    return f"{endpoint.rstrip('/')}/{bucket}"
+
+
+def target_id(target: S3Target) -> str:
+    return destination_id(target.endpoint, target.bucket)
+
+
+def blob_key(sha256: str, path: str) -> str:
+    """The bucket key for a file: `media/ab/cd/<sha256><ext>`. An extension
+    that isn't short and plain is left off, so a key is always safe."""
+    ext = Path(path).suffix.lower()
+    return f"{PREFIX}{sha256[:2]}/{sha256[2:4]}/{sha256}{ext if _EXT.match(ext) else ''}"
+
+
+# Every library file once per sha256: images and attachments together. An
+# attachment's sha can repeat across posts, and an image extracted from an
+# archive is its own file with its own sha.
+_LIBRARY = """
+    SELECT sha256, path, size_bytes, mime FROM image_record
+    UNION ALL
+    SELECT sha256, path, size_bytes, mime FROM post_attachment
+"""
+
+
+def pending(session, tid: str, *, after: str = "", limit: int = 500) -> list[Blob]:
+    """The next `limit` files not yet in the bucket, in sha order after
+    `after`, so a run walks the library once even when some files fail."""
+    rows = session.execute(text(f"""
+        SELECT DISTINCT ON (lib.sha256) lib.sha256, lib.path, lib.size_bytes, lib.mime
+        FROM ({_LIBRARY}) AS lib
+        WHERE lib.sha256 > :after
+          AND NOT EXISTS (
+              SELECT 1 FROM backup_blob b WHERE b.target = :tid AND b.sha256 = lib.sha256
+          )
+        ORDER BY lib.sha256, lib.path
+        LIMIT :limit
+    """), {"after": after, "tid": tid, "limit": limit}).all()
+    return [Blob(r.sha256, r.path, r.size_bytes, r.mime) for r in rows]
+
+
+def coverage(session, tid: str | None) -> dict:
+    """How much of the library is in the bucket: files and bytes, sent and
+    total, each sha counted once."""
+    row = session.execute(text(f"""
+        WITH lib AS (
+            SELECT DISTINCT ON (sha256) sha256, size_bytes FROM ({_LIBRARY}) AS u
+            ORDER BY sha256
+        )
+        SELECT count(*) AS files_total,
+               coalesce(sum(lib.size_bytes), 0) AS bytes_total,
+               count(b.sha256) AS files_sent,
+               coalesce(sum(lib.size_bytes) FILTER (WHERE b.sha256 IS NOT NULL), 0)
+                   AS bytes_sent
+        FROM lib
+        LEFT JOIN backup_blob b ON b.sha256 = lib.sha256 AND b.target = :tid
+    """), {"tid": tid or ""}).one()
+    return {
+        "files_total": row.files_total, "bytes_total": int(row.bytes_total),
+        "files_sent": row.files_sent, "bytes_sent": int(row.bytes_sent),
+    }
+
+
+def send(target: S3Target, blob: Blob, images_root: Path, pacer: Pacer | None) -> str:
+    """Upload one file to its key and return the key. Raises S3Error (or
+    OSError) with the reason when it can't be sent."""
+    path = Path(blob.path)
+    if not path.is_absolute():
+        path = images_root / path
+    if not path.is_file():
+        raise S3Error(f"{path} is not on disk.")
+    size = path.stat().st_size
+    if size != blob.size_bytes:
+        raise S3Error(f"{path} is {size} bytes on disk but {blob.size_bytes} when imported.")
+    key = blob_key(blob.sha256, blob.path)
+    content_type = blob.mime or "application/octet-stream"
+    try:
+        if size > MULTIPART_OVER:
+            put_file_in_parts(target, key, path, sha256=blob.sha256,
+                              content_type=content_type, pacer=pacer)
+        else:
+            put_file(target, key, path, content_type=content_type,
+                     payload_sha256=blob.sha256, pacer=pacer)
+    except S3Error:
+        if object_present(target, key, size):
+            return key          # already there from an earlier run
+        raise
+    return key
+
+
+def record(session, tid: str, blobs: list[tuple[Blob, str]]) -> None:
+    """Note files as sent: (blob, key) pairs. A row already there is kept."""
+    if blobs:
+        session.execute(insert(BackupBlob).values([
+            {"target": tid, "sha256": b.sha256, "key": key, "size_bytes": b.size_bytes}
+            for b, key in blobs
+        ]).on_conflict_do_nothing())
+
+
+def reconcile(session, target: S3Target) -> dict:
+    """Check backup_blob against what the bucket actually holds under
+    `media/`. A row whose object is missing or the wrong size is dropped, so
+    the next run sends that file again; an object the bucket holds for a
+    library file with no row (a restored database, a lost commit) is recorded,
+    so it isn't sent twice. Returns the counts."""
+    tid = target_id(target)
+    held: dict[str, int] = {}
+    token = None
+    while True:
+        page = list_objects(target, PREFIX, LIST_PAGE, continuation=token)
+        held.update((o.key, o.size) for o in page.objects)
+        token = page.next_token
+        if not token:
+            break
+        time.sleep(LIST_REST_SECONDS)
+
+    rows = session.execute(
+        select(BackupBlob.sha256, BackupBlob.key, BackupBlob.size_bytes)
+        .where(BackupBlob.target == tid)
+    ).all()
+    gone = [r.sha256 for r in rows if held.get(r.key) != r.size_bytes]
+    for at in range(0, len(gone), 1000):
+        session.execute(delete(BackupBlob).where(
+            BackupBlob.target == tid, BackupBlob.sha256.in_(gone[at:at + 1000]),
+        ))
+    kept = {r.sha256 for r in rows} - set(gone)
+
+    found = []
+    for r in session.execute(text(f"""
+        SELECT DISTINCT ON (sha256) sha256, path, size_bytes, mime
+        FROM ({_LIBRARY}) AS lib ORDER BY sha256, path
+    """)).all():
+        if r.sha256 in kept:
+            continue
+        blob = Blob(r.sha256, r.path, r.size_bytes, r.mime)
+        key = blob_key(blob.sha256, blob.path)
+        if held.get(key) == blob.size_bytes:
+            found.append((blob, key))
+    for at in range(0, len(found), 1000):
+        record(session, tid, found[at:at + 1000])
+    session.commit()
+    return {"listed": len(held), "dropped": len(gone), "adopted": len(found)}

backend/app/services/backup_s3.py

@@ -0,0 +1,505 @@
+"""A small S3 client for sending backups off the box: AWS Signature Version 4,
+path-style addressing (endpoint/bucket/key), and the calls a backup needs —
+put an object (whole, or in parts when its size isn't known up front), read
+one back, list a prefix.
+
+FabledCurator puts, lists and reads; it never deletes. The bucket keeps the
+history and thins it on its own policy, so a key that may only add and read
+objects is enough, and is the safer key to hand a service: a stolen one cannot
+wipe the history it writes into.
+
+Built on the standard library's http.client rather than boto3: three calls
+don't justify a large dependency, and signing is a pure function the tests
+hold to AWS's own published examples. Every call is synchronous, because the
+uploads run in Celery workers; the API runs the Test destination call in a
+thread.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import http.client
+import os
+import socket
+import ssl
+import time
+import xml.etree.ElementTree as ET
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
+
+EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
+TIMEOUT_SECONDS = 30.0
+# A single object can be a multi-gigabyte video or archive; a slow link still
+# gets this long between bytes before the call gives up.
+TRANSFER_TIMEOUT_SECONDS = 15 * 60
+CHUNK = 1 << 20
+
+
+class S3Error(Exception):
+    """Why a call to the bucket failed, said to the person."""
+
+
+@dataclass(frozen=True)
+class S3Target:
+    endpoint: str          # http(s)://host[:port][/prefix], no bucket
+    bucket: str
+    access_key: str
+    secret_key: str
+    region: str = "us-east-1"
+    verify_tls: bool = True
+
+    def url(self, key: str = "") -> str:
+        base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}"
+        return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/"
+
+
+@dataclass(frozen=True)
+class StoredObject:
+    key: str
+    size: int
+    last_modified: str
+
+
+@dataclass(frozen=True)
+class ListPage:
+    objects: list[StoredObject]
+    next_token: str | None     # pass back as `continuation` for the next page
+
+
+# ── Signing ──────────────────────────────────────────────────────────────────
+
+def _encode(text: str, keep_slash: bool = False) -> str:
+    return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
+
+
+def _hmac(key: bytes, msg: str) -> bytes:
+    return hmac.new(key, msg.encode(), hashlib.sha256).digest()
+
+
+def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
+         access_key: str, secret_key: str, region: str, at: datetime,
+         service: str = "s3") -> dict[str, str]:
+    """The request's headers with Host, x-amz-date, x-amz-content-sha256 and
+    Authorization added. Every header given is signed, Host always."""
+    u = urlsplit(url)
+    stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
+    day = stamp[:8]
+    out = dict(headers)
+    out["Host"] = u.netloc
+    out["x-amz-date"] = stamp
+    out["x-amz-content-sha256"] = payload_sha256
+    signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
+    names = sorted(signed)
+
+    query = sorted(
+        (_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True)
+    )
+    canonical = "\n".join([
+        method.upper(),
+        _encode(unquote(u.path) or "/", keep_slash=True),
+        "&".join(f"{k}={v}" for k, v in query),
+        "".join(f"{n}:{signed[n]}\n" for n in names),
+        ";".join(names),
+        payload_sha256,
+    ])
+    scope = f"{day}/{region}/{service}/aws4_request"
+    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
+                         hashlib.sha256(canonical.encode()).hexdigest()])
+    key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
+                      service), "aws4_request")
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
+                            f"SignedHeaders={';'.join(names)}, Signature={signature}")
+    return out
+
+
+# ── Transport ────────────────────────────────────────────────────────────────
+
+def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection:
+    u = urlsplit(url)
+    if u.scheme == "https":
+        ctx = ssl.create_default_context()
+        if not target.verify_tls:
+            ctx.check_hostname = False
+            ctx.verify_mode = ssl.CERT_NONE
+        return http.client.HTTPSConnection(
+            u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK,
+        )
+    if u.scheme != "http":
+        raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.")
+    return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK)
+
+
+def _request_target(url: str) -> str:
+    u = urlsplit(url)
+    return (u.path or "/") + (f"?{u.query}" if u.query else "")
+
+
+def _reach_error(target: S3Target, exc: Exception) -> S3Error:
+    if isinstance(exc, TimeoutError):
+        return S3Error(f"{target.endpoint} didn't answer in time.")
+    if isinstance(exc, socket.gaierror):
+        return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.")
+    if isinstance(exc, ConnectionRefusedError):
+        return S3Error(
+            f"Couldn't reach {target.endpoint}: nothing is listening there "
+            "(check the port)."
+        )
+    if isinstance(exc, ssl.SSLError):
+        return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).")
+    return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
+
+
+def _says(status: int, reason: str, content: bytes) -> str:
+    """An S3 error response in words: its Code and Message when it has them."""
+    try:
+        root = ET.fromstring(content) if content else None
+        code = (root.findtext("Code") or "") if root is not None else ""
+        message = (root.findtext("Message") or "") if root is not None else ""
+    except ET.ParseError:
+        code, message = "", ""
+    hints = {
+        "SignatureDoesNotMatch": "the secret key is wrong",
+        "InvalidAccessKeyId": "the access key isn't known there",
+        "NoSuchBucket": "there is no such bucket",
+        "AccessDenied": "the key isn't allowed to do that",
+        "XAmzContentSHA256Mismatch": "the file's bytes don't match its recorded checksum",
+    }
+    said = hints.get(code) or message or reason
+    if not code and 400 <= status < 500:
+        # S3 refuses with an <Error><Code> body. A 4xx without one came from
+        # something else at that address — most often a web server on the
+        # default port, when the S3 service listens on a port of its own.
+        said = ("the address answered, but not as S3 — check the scheme and "
+                "the port (an S3 service usually listens on its own port)")
+    return f"HTTP {status}{f' {code}' if code else ''}: {said}."
+
+
+# ── Calls ────────────────────────────────────────────────────────────────────
+
+def file_sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(CHUNK):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def put_file(target: S3Target, key: str, path: Path, *,
+             content_type: str = "application/octet-stream",
+             payload_sha256: str | None = None,
+             pacer: Pacer | None = None,
+             now: datetime | None = None) -> None:
+    """Upload one file as one object, streamed from disk. `payload_sha256` is
+    the file's sha256 when the caller already knows it (the library stores
+    every file's), which saves reading the file twice; the bucket checks the
+    bytes against it, so a file that changed on disk is refused rather than
+    stored under the wrong name."""
+    digest = payload_sha256 or file_sha256(path)
+    size = path.stat().st_size
+    url = target.url(key)
+    headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type},
+                   digest, target.access_key, target.secret_key, target.region,
+                   now or datetime.now(UTC))
+    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
+    try:
+        with open(path, "rb") as f:
+            conn.request("PUT", _request_target(url),
+                         body=pacer.file_chunks(f) if pacer else f, headers=headers)
+            response = conn.getresponse()
+            content = response.read()
+    except (OSError, http.client.HTTPException) as exc:
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+    if response.status // 100 != 2:
+        raise S3Error(f"Upload of {key} refused: "
+                      f"{_says(response.status, response.reason, content)}")
+
+
+def get_file(target: S3Target, key: str, path: Path, *,
+             now: datetime | None = None) -> None:
+    """Download one object to `path`, streamed to disk. The bytes land under a
+    `.partial` name and are renamed into place when complete, so a broken
+    transfer never leaves a truncated file at `path`."""
+    url = target.url(key)
+    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
+                   target.region, now or datetime.now(UTC))
+    partial = path.with_name(path.name + ".partial")
+    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
+    try:
+        conn.request("GET", _request_target(url), headers=headers)
+        response = conn.getresponse()
+        if response.status != 200:
+            raise S3Error(f"Reading {key} refused: "
+                          f"{_says(response.status, response.reason, response.read())}")
+        with open(partial, "wb") as f:
+            while chunk := response.read(CHUNK):
+                f.write(chunk)
+        os.replace(partial, path)
+    except (OSError, http.client.HTTPException) as exc:
+        partial.unlink(missing_ok=True)
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+
+
+def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
+                 continuation: str | None = None,
+                 now: datetime | None = None) -> ListPage:
+    """One page (up to `limit`) of the objects under `prefix`, ListObjectsV2."""
+    params = {"list-type": "2", "max-keys": str(limit)}
+    if prefix:
+        params["prefix"] = prefix
+    if continuation:
+        params["continuation-token"] = continuation
+    url = f"{target.url()}?{urlencode(params, quote_via=quote)}"
+    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
+                   target.region, now or datetime.now(UTC))
+    conn = _connect(target, url, TIMEOUT_SECONDS)
+    try:
+        conn.request("GET", _request_target(url), headers=headers)
+        response = conn.getresponse()
+        content = response.read()
+    except (OSError, http.client.HTTPException) as exc:
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+    if response.status != 200:
+        raise S3Error(f"Listing {target.bucket} refused: "
+                      f"{_says(response.status, response.reason, content)}")
+    try:
+        root = ET.fromstring(content)
+    except ET.ParseError:
+        raise S3Error(
+            f"{target.endpoint} answered a listing with something that isn't S3's XML."
+        ) from None
+    objects, next_token = [], None
+    for item in root.iter():
+        tag = item.tag.rsplit("}", 1)[-1]
+        if tag == "NextContinuationToken":
+            next_token = item.text or None
+        if tag != "Contents":
+            continue
+        fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
+        objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
+                                    fields.get("LastModified", "")))
+    return ListPage(objects, next_token)
+
+
+# ── Multipart: one object sent in parts ─────────────────────────────────────
+
+# Each part is held in memory while it is sent, so this bounds the memory a
+# streamed upload uses. S3 allows 10,000 parts, which at this size is 640 GiB
+# for one object, and every part but the last must be at least 5 MiB.
+PART_SIZE = 64 * CHUNK
+# A part the bucket couldn't take (it didn't answer, or said it was busy) is
+# sent again after a rest that grows each time: 15 s, then 60 s.
+PART_RETRY_WAITS = (15, 60)
+_BUSY = frozenset({429, 500, 502, 503, 504})
+_MAX_RETRY_AFTER = 300
+
+
+class Pacer:
+    """Holds the bytes a job sends to an average rate, across every request
+    it makes, so a backup leaves room on the link for everything else."""
+
+    def __init__(self, mbps: int | None):
+        self.rate = mbps * 1_000_000 / 8 if mbps else None   # bytes per second
+        self.sent = 0
+        self.started = time.monotonic()
+
+    def _wait(self, n: int) -> None:
+        if self.rate:
+            self.sent += n
+            ahead = self.sent / self.rate - (time.monotonic() - self.started)
+            if ahead > 0:
+                time.sleep(ahead)
+
+    def chunks(self, data: bytes) -> Iterator[memoryview]:
+        view = memoryview(data)
+        for at in range(0, len(view), CHUNK):
+            piece = view[at:at + CHUNK]
+            self._wait(len(piece))
+            yield piece
+
+    def file_chunks(self, f) -> Iterator[bytes]:
+        while piece := f.read(CHUNK):
+            self._wait(len(piece))
+            yield piece
+
+
+def _send(target: S3Target, method: str, url: str, body: bytes = b"",
+          headers: dict[str, str] | None = None, *, timeout: float = TIMEOUT_SECONDS,
+          pacer: Pacer | None = None,
+          now: datetime | None = None) -> tuple[int, str, http.client.HTTPMessage, bytes]:
+    """Sign and send one request with an in-memory body, paced when a pacer
+    is given. Returns (status, reason, headers, content)."""
+    digest = hashlib.sha256(body).hexdigest()
+    signed = sign(method, url, {"Content-Length": str(len(body)), **(headers or {})},
+                  digest, target.access_key, target.secret_key, target.region,
+                  now or datetime.now(UTC))
+    conn = _connect(target, url, timeout)
+    try:
+        conn.request(method, _request_target(url),
+                     body=pacer.chunks(body) if pacer and body else body, headers=signed)
+        response = conn.getresponse()
+        content = response.read()
+    except (OSError, http.client.HTTPException) as exc:
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+    return response.status, response.reason, response.headers, content
+
+
+def _xml_field(content: bytes, name: str) -> str | None:
+    try:
+        root = ET.fromstring(content)
+    except ET.ParseError:
+        return None
+    for item in root.iter():
+        if item.tag.rsplit("}", 1)[-1] == name:
+            return item.text
+    return None
+
+
+def _is_error_body(content: bytes) -> bool:
+    try:
+        return bool(content) and ET.fromstring(content).tag.rsplit("}", 1)[-1] == "Error"
+    except ET.ParseError:
+        return False
+
+
+def _retry_after(headers: http.client.HTTPMessage | None, default: int) -> int:
+    try:
+        asked = int((headers or {}).get("Retry-After") or default)
+    except ValueError:
+        asked = default
+    return max(0, min(asked, _MAX_RETRY_AFTER))
+
+
+class Upload:
+    """One object written as a series of parts, for data whose size isn't known
+    when it starts, such as a database dump streamed from pg_dump.
+
+    The object appears in the bucket only when `complete()` succeeds. Until
+    then the parts are invisible, so a stream that fails partway leaves
+    nothing behind that looks like a finished backup. `abort()` asks the
+    bucket to drop the parts at once; if the abort is refused or never
+    arrives, S3 services expire an unfinished upload on their own (Fabled
+    Archives after a day)."""
+
+    def __init__(self, target: S3Target, key: str, *,
+                 content_type: str = "application/octet-stream",
+                 pacer: Pacer | None = None):
+        self.target = target
+        self.key = key
+        self.size = 0
+        self.pacer = pacer
+        self._etags: list[str] = []
+        status, reason, _, content = _send(
+            target, "POST", f"{target.url(key)}?uploads", headers={"Content-Type": content_type},
+        )
+        upload_id = _xml_field(content, "UploadId") if status == 200 else None
+        if not upload_id:
+            raise S3Error(f"Starting the upload of {key} refused: "
+                          f"{_says(status, reason, content)}")
+        self.upload_id = upload_id
+
+    def _url(self, **params: str) -> str:
+        query = urlencode({**params, "uploadId": self.upload_id}, quote_via=quote)
+        return f"{self.target.url(self.key)}?{query}"
+
+    @property
+    def parts(self) -> int:
+        return len(self._etags)
+
+    def add(self, data: bytes) -> None:
+        """Send the next part, resting and sending it again when the bucket
+        doesn't answer or says it is busy."""
+        n = len(self._etags) + 1
+        for wait in (*PART_RETRY_WAITS, None):
+            headers = None
+            try:
+                status, reason, headers, content = _send(
+                    self.target, "PUT", self._url(partNumber=str(n)), data,
+                    timeout=TRANSFER_TIMEOUT_SECONDS, pacer=self.pacer,
+                )
+            except S3Error as exc:
+                if wait is None:
+                    raise S3Error(f"Part {n} of {self.key}: {exc}") from None
+                time.sleep(wait)
+                continue
+            etag = headers.get("ETag") if status == 200 else None
+            if etag:
+                self._etags.append(etag)
+                self.size += len(data)
+                return
+            if status not in _BUSY or wait is None:
+                raise S3Error(f"Part {n} of {self.key} refused: "
+                              f"{_says(status, reason, content)}")
+            time.sleep(_retry_after(headers, wait))
+
+    def complete(self) -> None:
+        """Join the parts into the object. S3 can refuse a completion with an
+        error body under a 200 status, so the body is read too."""
+        if not self._etags:
+            raise S3Error(f"Nothing was sent for {self.key}.")
+        body = "<CompleteMultipartUpload>" + "".join(
+            f"<Part><PartNumber>{n}</PartNumber><ETag>{etag}</ETag></Part>"
+            for n, etag in enumerate(self._etags, start=1)
+        ) + "</CompleteMultipartUpload>"
+        status, reason, _, content = _send(
+            self.target, "POST", self._url(), body.encode(),
+            headers={"Content-Type": "application/xml"}, timeout=TRANSFER_TIMEOUT_SECONDS,
+        )
+        if status != 200 or _is_error_body(content):
+            raise S3Error(f"Finishing the upload of {self.key} refused: "
+                          f"{_says(status, reason, content)}")
+
+    def abort(self) -> None:
+        """Drop the parts sent so far. Never raises: it runs while another
+        failure is already being reported, and the bucket expires the parts on
+        its own if this doesn't reach it."""
+        try:
+            _send(self.target, "DELETE", self._url())
+        except Exception:  # noqa: BLE001 — best effort, see above
+            pass
+
+
+def put_file_in_parts(target: S3Target, key: str, path: Path, *, sha256: str,
+                      content_type: str = "application/octet-stream",
+                      pacer: Pacer | None = None) -> int:
+    """Upload a large file as a multipart object, read from disk a part at a
+    time. The whole file's sha256 is computed on the way and must equal
+    `sha256` before the upload is completed; otherwise it is aborted, so a file
+    that changed on disk is never stored under its old name. Returns the
+    number of bytes sent."""
+    upload = Upload(target, key, content_type=content_type, pacer=pacer)
+    digest = hashlib.sha256()
+    try:
+        with open(path, "rb") as f:
+            while part := f.read(PART_SIZE):
+                digest.update(part)
+                upload.add(part)
+        if digest.hexdigest() != sha256:
+            raise S3Error(f"{path} no longer matches its recorded checksum; not stored.")
+        upload.complete()
+    except BaseException:
+        upload.abort()
+        raise
+    return upload.size
+
+
+def object_present(target: S3Target, key: str, size: int) -> bool:
+    """Whether `key` is already in the bucket at `size` bytes. Used when a put
+    is refused: an append-only bucket refuses to replace an object that is
+    already there, which for a content-addressed key means the work is done."""
+    try:
+        page = list_objects(target, prefix=key, limit=1)
+    except S3Error:
+        return False
+    return any(o.key == key and o.size == size for o in page.objects)

backend/app/services/backup_service.py

@@ -4,7 +4,9 @@ Two independent backup kinds:
   - 'db'     — pg_dump only; fast; nightly via Beat (settings-gated)
   - 'images' — tar+zstd of /images; slow; manual trigger only
 
-Files live under <images_root>/_backups/. Each backup writes:
+With an off-site destination set up, a database dump streams straight to that
+bucket instead (backup_db_offsite). Otherwise files live under
+<images_root>/_backups/. Each local backup writes:
   fc_<kind>_<ts>.{sql|tar.zst}        — the artifact
   fc_<kind>_<ts>.json                 — manifest (kind/tag/triggered_by)
 
@@ -19,6 +21,7 @@ import os
 import shutil
 import subprocess
 import tempfile
+import time
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -153,10 +156,10 @@ def backup_db(
     tmp_path = Path(tmp_name)
     try:
         _run_bounded(
-            [
+            _gently([
                 "pg_dump", "--no-owner", "--no-acl", "-Fc",
                 "-f", str(tmp_path), _libpq_url(db_url),
-            ],
+            ]),
             _DB_SUBPROCESS_TIMEOUT_S,
         )
         shutil.move(str(tmp_path), str(sql_path))
@@ -177,6 +180,79 @@ def backup_db(
     }
 
 
+class DumpFailed(Exception):
+    """pg_dump exited with an error; the message is its last line of output."""
+
+
+def offsite_dump_key(ts: str) -> str:
+    """Where a database dump lands in the off-site bucket. Dated names sit
+    side by side under one prefix, which is what lets the bucket's retention
+    thin them by age."""
+    return f"database/fabledcurator-{ts}.dump"
+
+
+def _gently(cmd: list[str]) -> list[str]:
+    """`cmd` at low CPU priority, and low disk priority where `ionice` is
+    installed, so a long dump leaves the server responsive."""
+    io = ["ionice", "-c", "2", "-n", "7"] if shutil.which("ionice") else []
+    return ["nice", "-n", "10", *io, *cmd]
+
+
+def _last_line(f) -> str | None:
+    f.seek(0)
+    lines = f.read().decode(errors="replace").strip().splitlines()
+    return lines[-1].strip() if lines else None
+
+
+def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) -> dict:
+    """Stream `pg_dump` straight into the off-site bucket, in parts, with no
+    copy staged on this server. The object appears only once pg_dump has
+    exited cleanly and every part has arrived; a dump that fails partway
+    leaves nothing in the bucket, and raises DumpFailed with pg_dump's own
+    last line. Returns a dict for the caller to record in BackupRun."""
+    from .backup_s3 import PART_SIZE, Pacer, Upload
+
+    ts = _now_ts()
+    key = offsite_dump_key(ts)
+    deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.Popen(
+            _gently(["pg_dump", "--no-owner", "--no-acl", "-Fc", _libpq_url(db_url)]),
+            stdout=subprocess.PIPE, stderr=err,
+        )
+        upload = None
+        try:
+            upload = Upload(target, key, pacer=Pacer(limit_mbps))
+            while part := proc.stdout.read(PART_SIZE):
+                if time.monotonic() > deadline:
+                    raise subprocess.TimeoutExpired(proc.args, _DB_SUBPROCESS_TIMEOUT_S)
+                upload.add(part)
+            proc.stdout.close()
+            proc.wait(timeout=_KILL_REAP_GRACE_S * 6)
+            if proc.returncode != 0:
+                raise DumpFailed(_last_line(err) or f"pg_dump exited with {proc.returncode}")
+            upload.complete()
+        except BaseException:
+            if proc.poll() is None:
+                proc.kill()
+                try:
+                    proc.wait(timeout=_KILL_REAP_GRACE_S)
+                except subprocess.TimeoutExpired:
+                    pass  # unkillable (D-state): abandon the reap, fail fast
+            if upload is not None:
+                upload.abort()
+            raise
+    return {
+        "kind": "db",
+        "ts": ts,
+        "key": key,
+        "bucket": target.bucket,
+        "endpoint": target.endpoint,
+        "parts": upload.parts,
+        "size_bytes": upload.size,
+    }
+
+
 def backup_images(
     *, images_root: Path,
     tag: str | None = None, triggered_by: str = "manual",

backend/app/services/discord_client.py

@@ -32,6 +32,13 @@ Two deliberate departures, both about the walk order, neither about content:
     the walk. gallery-dl skips only nested channels; one private thread the
     token cannot read would otherwise stop every channel after it.
 
+And one about content: a message is taken only if it has an image or video
+ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
+as post, and the operator wants the art, not the conversation (2026-09-28): a
+text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
+chat. A message that passes keeps every file gallery-dl would take, numbered
+as gallery-dl numbers them, so the on-disk names still match.
+
 FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 """
 
@@ -68,6 +75,9 @@ _THREADS_BATCH = 25
 # it is honoured when present; 60s is the fallback and the cap.
 _MAX_429_RETRIES = 4
 _429_WAIT_SECONDS = 60.0
+# How far back the poster picker looks: two pages. Enough to see who posts in
+# a channel, few enough to answer while the operator waits (#4491).
+_POSTER_SCAN = 200
 
 # https://discord.com/developers/docs/resources/message#message-object-message-types
 # DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
@@ -79,6 +89,13 @@ _FORUM = frozenset({15, 16})            # forum, media: threads only
 _CATEGORY = 4
 _SERVER_WALK = _TEXT | _FORUM
 _EMBED_TYPES = frozenset({"image", "gifv", "video"})
+# What makes a message art rather than chat: an attached file FC imports as an
+# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
+# import pipeline, and this one is a network client).
+_VISUAL_EXTS = frozenset({
+    "png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
+    "mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
+})
 
 _URL_RE = re.compile(
     r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
@@ -163,6 +180,82 @@ def message_text(message: dict) -> str:
     return "\n".join(p for p in parts if p)
 
 
+def _message_and_snapshots(message: dict) -> list[dict]:
+    """The message itself, then each forwarded message it carries."""
+    return [message] + [
+        (s or {}).get("message") or {}
+        for s in message.get("message_snapshots") or []
+        if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
+    ]
+
+
+def _is_visual(attachment: dict) -> bool:
+    ctype = (attachment.get("content_type") or "").lower()
+    if ctype.startswith(("image/", "video/")):
+        return True
+    name = attachment.get("filename") or attachment.get("url") or ""
+    return nameext_from_url(name)[1] in _VISUAL_EXTS
+
+
+def has_visual_attachment(message: dict) -> bool:
+    """Does this message, or a message it forwards, have an image or video
+    attached? The one test for "art, not chat" — see the module docstring."""
+    return any(
+        att.get("url") and _is_visual(att)
+        for snap in _message_and_snapshots(message)
+        for att in snap.get("attachments") or []
+    )
+
+
+_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
+# Words a server name wraps around its creator's: "Todding's Server",
+# "The Official Todding Discord".
+_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
+
+
+def _squash(text: str | None) -> str:
+    return _SQUASH.sub("", (text or "").lower())
+
+
+def _server_tokens(server_name: str | None) -> list[str]:
+    name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
+    words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
+    tokens = [w for w in words if len(w) >= 3]
+    joined = "".join(words)
+    if len(joined) >= 3 and joined not in tokens:
+        tokens.append(joined)
+    return tokens
+
+
+def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
+    """Mark who is probably the creator a server is named for, and order the
+    list for picking: suggested first, then by images posted, then messages.
+
+    Two signals, each a reason the picker shows beside the name:
+      - they own the server;
+      - their username or display name matches the server's name, once
+        "'s Server", "Official", "Discord" and punctuation are taken off
+        ("Todding's Server" and "todding").
+    Neither signal alone is proof, which is why this suggests and never
+    decides: the operator ticks the list. Bots are never suggested."""
+    tokens = _server_tokens(server_name)
+    owner = str(owner_id) if owner_id else None
+    ranked = []
+    for p in posters:
+        reasons = []
+        if owner and p.get("id") == owner:
+            reasons.append("owns the server")
+        names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
+        if any(t in n or n in t for t in tokens for n in names):
+            reasons.append("name matches the server")
+        ranked.append({**p, "reasons": reasons, "suggested": False})
+    best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
+    for r in ranked:
+        r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
+    ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
+    return ranked
+
+
 @dataclass
 class MediaItem:
     """One file of a Discord message. `filename`/`extension` are gallery-dl's
@@ -212,6 +305,8 @@ class DiscordClient:
         self._server: dict = {}
         self._channels: dict[str, dict] = {}
         self._skip_feed = False
+        # Whose messages the walk takes (`only_from`); empty takes everyone's.
+        self._authors: frozenset[str] = frozenset()
 
     # -- request -----------------------------------------------------------
 
@@ -373,6 +468,43 @@ class DiscordClient:
             if meta["channel_type"] in _SERVER_WALK:
                 yield from self._feeds(meta["channel_id"], safe=True)
 
+    def only_from(self, authors) -> None:
+        """Take only messages posted by these people: user ids, usernames or
+        display names, any case. A creator's server is full of other members
+        posting their own pictures; the source is subscribed to the creator.
+        Empty or None takes everyone's, as before."""
+        self._authors = frozenset(
+            str(a).strip().lower() for a in authors or () if str(a).strip()
+        )
+
+    def _author_wanted(self, message: dict) -> bool:
+        if not self._authors:
+            return True
+        author = message.get("author") or {}
+        names = (author.get("id"), author.get("username"), author.get("global_name"))
+        return any(str(n).lower() in self._authors for n in names if n)
+
+    def source_label(self, url: str) -> str | None:
+        """What the source is called in Discord — `Server · #channel`, a
+        thread as `#parent › thread`, a whole server by its name — from the
+        metadata the walk loaded. None when the walk never got that far."""
+        try:
+            _, channel_id = parse_source_url(url)
+        except DiscordAPIError:
+            return None
+        server = self._server.get("server") or None
+        if channel_id is None:
+            return server
+        meta = self._channels.get(channel_id) or {}
+        name = meta.get("channel")
+        if not name:
+            return None
+        if name != "DMs":
+            name = f"#{name}"
+            if meta.get("is_thread") and meta.get("parent"):
+                name = f"#{meta['parent']} › {meta['channel']}"
+        return f"{server} · {name}" if server else name
+
     def skip_feed(self) -> None:
         """Optional core seam (#4413): end the current channel and go on to the
         next one. A tick's early-out means THIS channel has nothing new, not
@@ -440,6 +572,8 @@ class DiscordClient:
             for message in messages:
                 if message.get("type") not in MESSAGE_TYPES:
                     continue
+                if not self._author_wanted(message):
+                    continue
                 message["_meta"] = meta
                 yield message, meta, page_cursor
                 if self._skip_feed:
@@ -454,13 +588,14 @@ class DiscordClient:
     def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
         """gallery-dl's file list for one message: attachments, then the first
         of video/image/thumbnail `proxy_url` of each file-bearing embed, then
-        the same for every forwarded snapshot; numbered from 1 across them."""
+        the same for every forwarded snapshot; numbered from 1 across them.
+
+        Empty for a message with no image or video attached: that is chat,
+        whatever else it carries (`has_visual_attachment`)."""
+        if not has_visual_attachment(post):
+            return []
         mid = str(post.get("id") or "")
-        snapshots = [post] + [
-            (s or {}).get("message") or {}
-            for s in post.get("message_snapshots") or []
-            if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
-        ]
+        snapshots = _message_and_snapshots(post)
         found: list[tuple[str, str, str | None]] = []
         for snap in snapshots:
             for att in snap.get("attachments") or []:
@@ -498,10 +633,10 @@ class DiscordClient:
         """`(message:<id>, <id>)` — gates the message record through the seen
         ledger, like `post:<id>` on the other platforms.
 
-        None for a message with no files. gallery-dl wrote a sidecar only
-        beside a file, so a text-only chat line never became a post, and the
-        drop grouping (discord_grouping) is built on that: a channel's chatter
-        recorded as posts would bury the drops it exists to surface."""
+        None for a message `extract_media` takes nothing from, i.e. one with
+        no image or video attached. The drop grouping (discord_grouping) is
+        built on that: a channel's chatter recorded as posts would bury the
+        drops it exists to surface."""
         mid = post.get("id")
         mid = str(mid) if mid is not None else ""
         if not mid or not cls.extract_media(post):
@@ -528,6 +663,61 @@ class DiscordClient:
                 pass
         return out
 
+    def recent_posters(
+        self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
+    ) -> dict:
+        """Who has posted lately in a channel, for picking a source's poster
+        list (#4491): the newest `max_messages` messages, tallied per author.
+        Deliberately shallow — it answers "who posts here", not "who ever did".
+        Every poster comes back with their stable id, which is what the list
+        stores: a username can change on a whim, an id can't."""
+        server: dict = {}
+        if server_id:
+            try:
+                server = self._get(f"/guilds/{server_id}") or {}
+            except DiscordAPIError:
+                server = {}
+        tally: dict[str, dict] = {}
+        before = None
+        seen = 0
+        while seen < max_messages:
+            page = self._get(
+                f"/channels/{channel_id}/messages",
+                {"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
+            )
+            if not isinstance(page, list) or not page:
+                break
+            for message in page:
+                seen += 1
+                if message.get("type") not in MESSAGE_TYPES:
+                    continue
+                author = message.get("author") or {}
+                aid = str(author.get("id") or "")
+                if not aid:
+                    continue
+                row = tally.setdefault(aid, {
+                    "id": aid,
+                    "username": author.get("username"),
+                    "global_name": author.get("global_name"),
+                    "bot": bool(author.get("bot")),
+                    "messages": 0,
+                    "images": 0,
+                })
+                row["messages"] += 1
+                if has_visual_attachment(message):
+                    row["images"] += 1
+            if len(page) < _MESSAGES_BATCH:
+                break
+            before = str(page[-1]["id"])
+        return {
+            "server": server.get("name") or None,
+            "owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
+            "scanned": seen,
+            "posters": rank_posters(
+                server.get("name"), server.get("owner_id"), list(tally.values()),
+            ),
+        }
+
     def verify_auth(self, url: str) -> tuple[bool | None, str]:
         """Is the token valid, and can it see what the source names?"""
         try:

backend/app/services/discord_downloader.py

@@ -195,6 +195,7 @@ class DiscordDownloader(BaseNativeDownloader):
             "parent": meta.get("parent"),
             "is_thread": meta.get("is_thread"),
             "author": author.get("username"),
+            "author_name": author.get("global_name"),
             "author_id": author.get("id"),
             "message": body,
             "date": post.get("timestamp"),

backend/app/services/discord_ingester.py

@@ -13,6 +13,11 @@ Two things differ from the cookie platforms:
     something; a Discord drop is routinely files and nothing else, so on
     Discord that is an ordinary backfill, not a broken parser.
 
+And two things only a Discord source has: `discord_authors` in its
+config_overrides limits it to the creator's own messages (a server's other
+members post pictures too), and it learns its own name — the server and
+channel it walks — into `source.display_name`.
+
 `campaign_id` is the source URL (a server, channel, thread or category link).
 FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 """
@@ -20,15 +25,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 from __future__ import annotations
 
 import asyncio
+import logging
 from collections.abc import Callable
 from pathlib import Path
 
-from ..models import DiscordFailedMedia, DiscordSeenMedia
+from sqlalchemy import select, update
+
+from ..models import DiscordFailedMedia, DiscordSeenMedia, Source
 from .discord_client import DiscordAPIError, DiscordClient, MediaItem
 from .discord_downloader import DiscordDownloader
 from .ingest_core import Ingester
 
+log = logging.getLogger(__name__)
+
 _LEDGER_KEY_MAX = 128
+# `config_overrides` key: the people whose messages this source takes (user
+# ids, usernames or display names). Absent or empty takes everyone's.
+AUTHORS_KEY = "discord_authors"
 
 
 def _ledger_key(media: MediaItem) -> str:
@@ -74,6 +87,46 @@ class DiscordIngester(Ingester):
             body_canary=False,
         )
 
+    def run(self, **kwargs):
+        """The core walk, bracketed by the two things only a Discord source
+        has: whose messages it takes, read before the walk, and the name of
+        what it walks, written after it (the walk is what loads that name)."""
+        source_id = kwargs["source_id"]
+        self.client.only_from(self._source_authors(source_id))
+        try:
+            return super().run(**kwargs)
+        finally:
+            self._record_display_name(source_id, kwargs["campaign_id"])
+
+    def _source_authors(self, source_id: int) -> list:
+        if self.session_factory is None:
+            return []
+        with self.session_factory() as session:
+            overrides = session.execute(
+                select(Source.config_overrides).where(Source.id == source_id)
+            ).scalar_one_or_none() or {}
+        authors = overrides.get(AUTHORS_KEY) or []
+        return authors if isinstance(authors, list) else []
+
+    def _record_display_name(self, source_id: int, url: str) -> None:
+        """Refreshed on every walk, never written once: a renamed channel
+        should read as its new name. A name the walk couldn't read leaves the
+        stored one alone, and a failure here never fails the walk."""
+        label = self.client.source_label(url)
+        if not label or self.session_factory is None:
+            return
+        try:
+            with self.session_factory() as session:
+                session.execute(
+                    update(Source)
+                    .where(Source.id == source_id)
+                    .where(Source.display_name.is_distinct_from(label))
+                    .values(display_name=label)
+                )
+                session.commit()
+        except Exception as exc:  # a name is decoration — never fail the walk
+            log.warning("Discord: couldn't record source %s's name: %s", source_id, exc)
+
 
 async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
     """The uniform `(ok, message)` probe: is the token valid, and can its

backend/app/services/discord_poster_cleanup.py

@@ -0,0 +1,243 @@
+"""Remove a Discord source's posts by people outside its poster list (#4486).
+
+A creator's server is full of other members posting their own pictures. The
+poster list (`discord_authors`, #4481) stops a walk taking them; this removes
+the ones taken before the list existed. The operator sets the list first, then
+previews what would go, then applies.
+
+## What goes
+
+A post of this source whose record names its poster (`author_id` / `author`,
+written by `DiscordDownloader.write_post_record`) and names someone NOT on the
+list. Two kinds of post are never touched:
+
+* a post whose record names no poster — a message recorded before the native
+  ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
+* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
+  below as a consequence, not matched as a poster's post.
+
+An image goes when every REAL post it belongs to is going. A link to a
+synthetic drop does not keep an image: the drop only references its members'
+images. An image also on a kept post — the creator re-posting a piece someone
+else shared — stays.
+
+A drop that absorbed a removed message is deleted outright. Deleting a drop is
+its undo (discord_grouping's honesty rule): its remaining members return to
+the feed and the grouping sweep regroups them on its next run, so no half-
+rebuilt drop keeps text or thumbnails from someone it no longer contains.
+
+Preview and apply spread the same predicates (rule 93, snippet #3087).
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from sqlalchemy import and_, delete, exists, func, or_, select
+from sqlalchemy.orm import Session, aliased
+
+from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
+from .cleanup_service import delete_images
+from .discord_ingester import AUTHORS_KEY
+
+log = logging.getLogger(__name__)
+
+PLATFORM = "discord"
+# The record keys that name a message's poster. `author_name` (the display
+# name) is only on records written after 2026-09-28.
+_POSTER_KEYS = ("author_id", "author", "author_name")
+
+
+class PosterCleanupError(ValueError):
+    """The source can't be cleaned this way (not Discord, or no list)."""
+
+
+def source_authors(source: Source) -> list[str]:
+    """The source's poster list, lowercased; empty means everyone's wanted."""
+    authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
+    if not isinstance(authors, list):
+        return []
+    return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
+
+
+def _poster(key: str):
+    return func.lower(Post.raw_metadata[key].as_string())
+
+
+def _has_poster():
+    return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
+
+
+def _real_post_conditions(source_id: int) -> list:
+    return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
+
+
+def _named_poster_conditions(source_id: int) -> list:
+    """A real post of this source whose record says who posted it."""
+    return [*_real_post_conditions(source_id), _has_poster()]
+
+
+def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
+    """The posts that go: a named poster none of whose names is on the list."""
+    return [
+        *_named_poster_conditions(source_id),
+        *(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
+    ]
+
+
+def _doomed_post_ids(source_id: int, authors: list[str]):
+    # correlate(None): used inside queries that are themselves over `post` (the
+    # drops, the delete), where auto-correlation would bind this to the outer
+    # row instead of scanning the table.
+    return (
+        select(Post.id)
+        .where(*_other_poster_post_conditions(source_id, authors))
+        .correlate(None)
+    )
+
+
+def _image_conditions(source_id: int, authors: list[str]) -> list:
+    """Images that belong to a removed post and to no kept real post."""
+    doomed = _doomed_post_ids(source_id, authors)
+    keeper = aliased(Post)
+    on_doomed = or_(
+        ImageRecord.primary_post_id.in_(doomed),
+        exists().where(
+            ImageProvenance.image_record_id == ImageRecord.id,
+            ImageProvenance.post_id.in_(doomed),
+        ),
+    )
+    on_kept = or_(
+        and_(
+            ImageRecord.primary_post_id.isnot(None),
+            ImageRecord.primary_post_id.notin_(doomed),
+        ),
+        exists().where(
+            ImageProvenance.image_record_id == ImageRecord.id,
+            ImageProvenance.post_id == keeper.id,
+            keeper.synthesized_by.is_(None),
+            keeper.id.notin_(doomed),
+        ),
+    )
+    return [on_doomed, ~on_kept]
+
+
+def _drop_conditions(source_id: int, authors: list[str]) -> list:
+    """The synthetic drops that absorbed a removed message."""
+    member = aliased(Post)
+    return [
+        Post.source_id == source_id,
+        Post.synthesized_by.isnot(None),
+        exists().where(
+            member.absorbed_by_post_id == Post.id,
+            member.id.in_(_doomed_post_ids(source_id, authors)),
+        ),
+    ]
+
+
+def _attachment_conditions(source_id: int, authors: list[str]) -> list:
+    return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
+
+
+def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
+    source = session.get(Source, source_id)
+    if source is None:
+        raise LookupError(source_id)
+    if source.platform != PLATFORM:
+        raise PosterCleanupError("Only a Discord source has posters to filter by.")
+    authors = source_authors(source)
+    if not authors:
+        raise PosterCleanupError(
+            "Set this source's 'Only posts by' list first — with no list, "
+            "every poster is wanted and nothing would be removed."
+        )
+    return source, authors
+
+
+def _count(session: Session, stmt) -> int:
+    return session.execute(stmt).scalar_one()
+
+
+def preview(session: Session, *, source_id: int) -> dict:
+    """What `apply` would remove, per poster, without touching anything."""
+    _, authors = _load(session, source_id)
+    who = func.coalesce(
+        Post.raw_metadata["author_name"].as_string(),
+        Post.raw_metadata["author"].as_string(),
+        Post.raw_metadata["author_id"].as_string(),
+    )
+    rows = session.execute(
+        select(who, func.count(Post.id))
+        .where(*_other_poster_post_conditions(source_id, authors))
+        .group_by(who)
+        .order_by(func.count(Post.id).desc())
+    ).all()
+    posters = [{"poster": name, "posts": n} for name, n in rows]
+    unknown = _count(session, select(func.count(Post.id)).where(
+        *_real_post_conditions(source_id), ~_has_poster(),
+    ))
+    return {
+        "authors": authors,
+        "posters": posters,
+        "posts": sum(p["posts"] for p in posters),
+        "images": _count(session, select(func.count(ImageRecord.id)).where(
+            *_image_conditions(source_id, authors))),
+        "attachments": _count(session, select(func.count(PostAttachment.id)).where(
+            *_attachment_conditions(source_id, authors))),
+        "drops": _count(session, select(func.count(Post.id)).where(
+            *_drop_conditions(source_id, authors))),
+        "unknown_posts": unknown,
+    }
+
+
+def confirm_token(projection: dict) -> str:
+    """What the operator's apply must echo: the preview it was shown, so a
+    list edited between preview and apply can't delete an unseen set."""
+    return f"remove-{projection['posts']}-posts-{projection['images']}-images"
+
+
+def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
+    """Remove them. Counts are taken before the deletes they describe — once
+    the rows are gone there is nothing left to count (snippet #3087 note 2)."""
+    projection = preview(session, source_id=source_id)
+    _, authors = _load(session, source_id)
+
+    image_ids = session.execute(
+        select(ImageRecord.id).where(*_image_conditions(source_id, authors))
+    ).scalars().all()
+    # Drops first-class: found BEFORE their members go, since the link that
+    # finds them is the member's absorbed_by_post_id.
+    drop_ids = session.execute(
+        select(Post.id).where(*_drop_conditions(source_id, authors))
+    ).scalars().all()
+
+    deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
+    # Attachments before posts: post_attachment.post_id is SET NULL, and a
+    # NULL-post attachment collides on its partial unique index (see
+    # cleanup_service.delete_artist_cascade).
+    attachments = session.execute(
+        delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
+    ).rowcount or 0
+    posts = session.execute(
+        Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
+    ).rowcount or 0
+    drops = 0
+    if drop_ids:
+        drops = session.execute(
+            Post.__table__.delete().where(Post.id.in_(drop_ids))
+        ).rowcount or 0
+    session.commit()
+    log.info(
+        "discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
+        "%d attachment(s), %d drop(s) removed",
+        source_id, authors, posts, deleted["images_deleted"], attachments, drops,
+    )
+    return {
+        **projection,
+        "posts_deleted": posts,
+        "images_deleted": deleted["images_deleted"],
+        "files_deleted": deleted["files_deleted"],
+        "attachments_deleted": attachments,
+        "drops_deleted": drops,
+    }

backend/app/services/download_service.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import time
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -39,7 +40,7 @@ from .gallery_dl import (
     walk_completed,
 )
 from .importer import Importer
-from .ingest_core import DEFAULT_REVISIT_DAYS
+from .ingest_core import DEFAULT_REVISIT_DAYS, PHASE3_DEADLINE_SECONDS
 from .platforms import auth_type_for
 from .scheduler_service import set_platform_cooldown
 
@@ -82,6 +83,7 @@ class DownloadService:
 
     async def download_source(self, source_id: int) -> int:
         """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
+        started = time.monotonic()
         setup = await self._phase1_setup(source_id)
         if setup["status"] in ("skipped", "in_flight"):
             return setup["event_id"]
@@ -174,6 +176,7 @@ class DownloadService:
 
         return await self._phase3_persist(
             ctx["event_id"], ctx, dl_result, resolved_campaign_id,
+            started=started,
         )
 
     async def _run_download(
@@ -268,6 +271,8 @@ class DownloadService:
         ctx: dict,
         dl_result,
         resolved_campaign_id: str | None,
+        *,
+        started: float | None = None,
     ) -> int:
         # Cache Patreon campaign id on the source.
         if resolved_campaign_id:
@@ -303,7 +308,23 @@ class DownloadService:
 
         loop = asyncio.get_running_loop()
 
-        for path_str in dl_result.written_paths or []:
+        # Phase 3 shares the task's soft limit with the walk, and an import's
+        # cost is only estimated there. Native platforms stop starting imports
+        # at the deadline: what is left stays on disk with no record, which the
+        # next chunk's walk imports. gallery-dl platforms have no such pickup,
+        # so they import everything.
+        deadline = (
+            started + PHASE3_DEADLINE_SECONDS
+            if started is not None and uses_native_ingester(ctx["platform"])
+            else None
+        )
+        written_paths = list(dl_result.written_paths or [])
+        deferred_from: int | None = None
+
+        for idx, path_str in enumerate(written_paths):
+            if deadline is not None and time.monotonic() >= deadline:
+                deferred_from = idx
+                break
             if path_str in (dl_result.quarantined_paths or []):
                 continue
             path = Path(path_str)
@@ -348,11 +369,8 @@ class DownloadService:
                     if do_embed:
                         embed_image.delay(img_id)
             elif result.status == "attached":
-                # Non-media or extracted archive captured as PostAttachment
-                # (FC-2d-iii). The canonical copy lives in the attachments
-                # store; the original download path is now redundant —
-                # mirror duplicate_hash cleanup so we don't keep two copies.
-                # Operator-flagged 2026-06-02 (Lustria OST zip).
+                # Non-media, or an archive with no new images, captured as a
+                # PostAttachment; the attachments store holds the copy.
                 import_summary["attached"] += 1
                 # An archive captured WITHOUT extracting any image carries a
                 # reason on result.error — record it so the event explains the
@@ -369,35 +387,13 @@ class DownloadService:
                     bytes_downloaded += path.stat().st_size  # noqa: ASYNC240
                 except OSError:
                     pass
-                try:
-                    path.unlink(missing_ok=True)  # noqa: ASYNC240
-                except OSError:
-                    pass
-            elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
-                "duplicate_hash", "duplicate_phash",
-            ):
-                import_summary["skipped"] += 1
-                try:
-                    path.unlink(missing_ok=True)  # noqa: ASYNC240
-                except OSError:
-                    pass
             elif result.status == "skipped":
-                # Soft skip (too_small, too_transparent, invalid_image) —
-                # the file just didn't qualify, not a download/ingest
-                # failure. Don't flag the run as error; the file stays
-                # on disk for operator inspection.
+                # A duplicate, or a file the import filters rejected. Neither
+                # is a download failure, so the run is not flagged.
                 import_summary["skipped"] += 1
             elif result.status == "failed":
-                # Hard failure (today only: archive probe crash/timeout).
-                # The original archive sits in /images/ as an orphan; the
-                # filesystem scanner would re-import and re-crash on the
-                # same file, so delete the source file and surface the
-                # error in import_summary. Audit 2026-06-02.
+                # Hard failure: an archive probe crashed or timed out.
                 import_summary["errors"] += 1
-                try:
-                    path.unlink(missing_ok=True)  # noqa: ASYNC240
-                except OSError:
-                    pass
             elif result.status == "refreshed":
                 # Currently unreachable from attach_in_place (the download
                 # path never runs in deep=True mode), but the importer's
@@ -406,6 +402,10 @@ class DownloadService:
                 import_summary["attached"] += 1
             else:
                 import_summary["errors"] += 1
+            # Remove the download when the library no longer needs it at this
+            # path: an attachment or archive now in the store, a duplicate, a
+            # rejected file. One policy for every download path.
+            self.importer.discard_after_attach(path, result)
 
         # Post-only records: media-less posts (pure-text) the native ingester
         # captured so the artist archive is complete. Upsert each (keyed on
@@ -427,8 +427,16 @@ class DownloadService:
         # has been through the importer. Had the run died before here they stay
         # unmarked, and the next walk imports them from disk (ingest_core).
         mark_seen = getattr(dl_result, "mark_seen_after_import", None)
+        deferred = written_paths[deferred_from:] if deferred_from is not None else []
         if mark_seen is not None:
-            await loop.run_in_executor(None, mark_seen)
+            if deferred:
+                await loop.run_in_executor(
+                    None, lambda: mark_seen(skip_paths=frozenset(deferred)),
+                )
+            else:
+                await loop.run_in_executor(None, mark_seen)
+        if deferred:
+            self._defer_unimported(dl_result, deferred_from, len(deferred))
 
         # #830 recapture: backfill source_filehash on EXISTING on-disk images so
         # their post-body inline <img src=CDN> remaps to the local copy. A
@@ -530,6 +538,33 @@ class DownloadService:
         await self.async_session.commit()
         return event_id
 
+    @staticmethod
+    def _defer_unimported(dl_result, first: int, count: int) -> None:
+        """Turn this chunk into a chunk boundary at the first file phase 3 left.
+
+        The next chunk resumes on the page that file came from, so its walk
+        meets every left file again, on disk with no record, and imports it. A
+        walk that reached its end has not finished while files wait, so a clean
+        result becomes PARTIAL; a real failure keeps its own error.
+        """
+        cursors = getattr(dl_result, "written_cursors", None) or []
+        if first < len(cursors):
+            dl_result.cursor = cursors[first]
+        dl_result.phase3_imported_any = first > 0
+        msg = (
+            f"Imports stopped at the chunk deadline: {count} file(s) left "
+            "for the next chunk"
+        )
+        dl_result.stdout = f"{dl_result.stdout}\n{msg}" if dl_result.stdout else msg
+        log.info("%s", msg)
+        if dl_result.error_type in (None, ErrorType.PARTIAL, ErrorType.TIMEOUT) or (
+            is_informational(dl_result.error_type)
+        ):
+            dl_result.success = False
+            dl_result.return_code = -1
+            dl_result.error_type = ErrorType.PARTIAL
+            dl_result.error_message = msg
+
     async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None:
         """Backfill state machine (plan #693, building on the cursor of #689).
 
@@ -597,9 +632,12 @@ class DownloadService:
         new_cursor = (
             dl_result.cursor if uses_native_ingester(ctx["platform"]) else None
         )
+        # Imports phase 3 finished before its deadline are progress even when
+        # the cursor went back to the page holding the files it left.
         advanced = bool(
             (new_cursor and new_cursor != old_cursor)
             or dl_result.files_downloaded > 0
+            or getattr(dl_result, "phase3_imported_any", False)
         )
         if advanced:
             if new_cursor:

backend/app/services/extension_service.py

@@ -23,6 +23,12 @@ log = logging.getLogger(__name__)
 
 # The probe runs while the chip is drawing; names that take longer are skipped.
 _NAME_LOOKUP_SECONDS = 6.0
+# The poster picker waits on two pages of a channel's messages (#4491).
+_POSTER_LOOKUP_SECONDS = 20.0
+
+# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this
+# keeps the name each id was picked under, for showing the list to a person.
+AUTHOR_LABELS_KEY = "discord_author_labels"
 
 
 class UnknownPlatformError(Exception):
@@ -37,6 +43,15 @@ class UnknownArtistError(Exception):
     """quick-add named an `artist_id` that does not exist."""
 
 
+class PosterLookupError(Exception):
+    """The Discord poster picker couldn't read the channel (no token, no
+    access, not a channel page, or Discord was slow)."""
+
+
+class UnknownSourceError(Exception):
+    """No source follows this Discord channel or its server."""
+
+
 # Mirrored byte-for-byte from extension/lib/platforms.js
 # PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
 # reviewers catch drift.
@@ -110,6 +125,7 @@ class ExtensionService:
         artist_id: int | None = None,
         artist_name: str | None = None,
         use_platform_name: bool = False,
+        discord_authors: list[dict] | None = None,
     ) -> dict:
         """Add `url` as a source. `artist_id` connects it to an existing
         artist, `artist_name` to that artist (created if new); with neither,
@@ -119,7 +135,10 @@ class ExtensionService:
         name is canon: a Patreon source added to an existing artist renames
         that artist to the creator's Patreon display name. Name only — the
         slug, and every path keyed off it, never moves (#130). Ignored on every
-        other platform, and when the name can't be read."""
+        other platform, and when the name can't be read.
+
+        `discord_authors` ([{id, name}], #4491) sets a Discord source's poster
+        list in the same step, on a new source or one already there."""
         platform, raw_slug = self._derive(url)
         url = canonical_source_url(platform, url, raw_slug)
         renamed_from = None
@@ -133,6 +152,8 @@ class ExtensionService:
             artist = (await self.session.execute(
                 select(Artist).where(Artist.id == existing.artist_id)
             )).scalar_one()
+            if platform == DISCORD and discord_authors is not None:
+                await self._apply_posters(existing, discord_authors)
             return self._shape(existing, artist, created_source=False, created_artist=False)
 
         if artist_id is not None:
@@ -154,11 +175,98 @@ class ExtensionService:
         source, created_source = await self._find_or_create_source(
             artist_id=artist.id, platform=platform, url=url,
         )
+        if platform == DISCORD and discord_authors is not None:
+            await self._apply_posters(source, discord_authors)
         shaped = self._shape(source, artist, created_source, created_artist)
         if renamed_from is not None:
             shaped["renamed_from"] = renamed_from
         return shaped
 
+    # -- Discord poster picker (#4491) ------------------------------------------
+
+    def _discord_page(self, url: str) -> tuple[str, str]:
+        platform, raw_slug = self._derive(url)
+        if platform != DISCORD:
+            raise InvalidUrlError("not a Discord page")
+        server_id, _, channel_id = raw_slug.partition("/")
+        if not channel_id:
+            raise PosterLookupError("Open a channel to see who posts in it.")
+        return server_id, channel_id
+
+    async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None:
+        """The source this channel's posts land on: the channel's own, else a
+        whole-server source that walks it."""
+        base = f"https://discord.com/channels/{server_id}"
+        return (
+            await self._existing_source(DISCORD, f"{base}/{channel_id}")
+            or await self._existing_source(DISCORD, base)
+        )
+
+    async def discord_posters(self, url: str) -> dict:
+        """Who posts in the channel `url` shows, newest `_POSTER_SCAN`
+        messages deep, with the likely creator marked — and, when a source
+        already follows it, the poster list that source has now."""
+        server_id, channel_id = self._discord_page(url)
+        import asyncio
+
+        from .credential_service import CredentialService
+        from .discord_client import DiscordAPIError, DiscordClient
+        token = None
+        if self._crypto is not None:
+            token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
+        if not token:
+            raise PosterLookupError("No Discord token is saved in FabledCurator.")
+        client = DiscordClient(token, max_retries=0)
+        loop = asyncio.get_running_loop()
+        try:
+            found = await asyncio.wait_for(
+                loop.run_in_executor(None, client.recent_posters, server_id, channel_id),
+                timeout=_POSTER_LOOKUP_SECONDS,
+            )
+        except TimeoutError as exc:
+            raise PosterLookupError("Discord took too long to answer; try again.") from exc
+        except DiscordAPIError as exc:
+            raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc
+        source = await self._source_for_page(server_id, channel_id)
+        co = (source.config_overrides or {}) if source is not None else {}
+        from .discord_ingester import AUTHORS_KEY
+        return {
+            **found,
+            "source_id": source.id if source is not None else None,
+            "selected": list(co.get(AUTHORS_KEY) or []),
+            # So someone on the list who hasn't posted lately still shows by
+            # name, and can be unticked.
+            "selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}),
+        }
+
+    async def set_discord_posters(self, url: str, authors: list[dict]) -> dict:
+        """Store the picked posters on the source that follows `url`'s channel."""
+        server_id, channel_id = self._discord_page(url)
+        source = await self._source_for_page(server_id, channel_id)
+        if source is None:
+            raise UnknownSourceError("No source follows this channel yet.")
+        return await self._apply_posters(source, authors)
+
+    async def _apply_posters(self, source: Source, authors: list[dict]) -> dict:
+        """Ids are what the list keys on — a username can change on a whim,
+        an id can't — and the name each was picked under rides beside it for
+        showing the list to a person. An empty pick clears the list, which
+        takes everyone's posts again."""
+        from .discord_ingester import AUTHORS_KEY
+        picked = [a for a in authors if str(a.get("id") or "").strip()]
+        co = dict(source.config_overrides or {})
+        if picked:
+            co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked]
+            co[AUTHOR_LABELS_KEY] = {
+                str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked
+            }
+        else:
+            co.pop(AUTHORS_KEY, None)
+            co.pop(AUTHOR_LABELS_KEY, None)
+        source.config_overrides = co or None
+        await self.session.commit()
+        return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])}
+
     async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
         """Rename `artist` to the Patreon display name; the old name when it
         changed, else None. Unreadable name → no rename, never the handle."""

backend/app/services/external_fetch.py

@@ -241,11 +241,19 @@ def _fetch_gdrive(url: str, dest_dir: Path, *, read_timeout: float,
 def _fetch_mega(url: str, dest_dir: Path, *, read_timeout: float,
                 total_timeout: float,
                 should_stop: Callable[[], bool]) -> FetchResult:
-    before = set(dest_dir.iterdir()) if dest_dir.exists() else set()
+    # A folder link keeps its subfolders, so files are found at any depth.
+    # Dotfiles are megatools' `.megatmp.*` staging names.
+    def _complete_files() -> set[Path]:
+        return {
+            p for p in dest_dir.rglob("*")
+            if p.is_file() and not p.name.startswith(".")
+        } if dest_dir.exists() else set()
+
+    before = _complete_files()
     # megatools is a subprocess: its timeout IS a total wall-clock cap (the read
     # timeout has no analogue here), so the total budget applies directly.
     _run_mega_get(url, str(dest_dir), timeout=total_timeout)
-    new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()]
+    new = sorted(_complete_files() - before)
     if not new:
         return FetchResult(error="mega-get wrote no new file")
     return FetchResult(files=new, bytes=sum(p.stat().st_size for p in new))

backend/app/services/external_links.py

@@ -21,9 +21,9 @@ log = logging.getLogger(__name__)
 
 
 def refetch_links_for_post(session: Session, post_id: int) -> dict:
-    """Reset every settled ExternalLink on a post to pending (fresh attempt
-    budget) and dispatch its fetch. In-flight ('downloading') links are left
-    alone. Commits. Returns {links_total, links_reset}."""
+    """Reset every settled ExternalLink on a post to pending (fresh attempt and
+    recovery budgets) and dispatch its fetch. In-flight ('downloading') links
+    are left alone. Commits. Returns {links_total, links_reset}."""
     links = session.execute(
         select(ExternalLink).where(ExternalLink.post_id == post_id)
     ).scalars().all()
@@ -33,6 +33,7 @@ def refetch_links_for_post(session: Session, post_id: int) -> dict:
             continue
         link.status = "pending"
         link.attempts = 0
+        link.recoveries = 0
         link.last_error = None
         link.completed_at = None
         reset_ids.append(link.id)

backend/app/services/gallery_dl.py

@@ -169,6 +169,10 @@ class DownloadResult:
     # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
     # on the gallery-dl path and outside recapture.
     relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
+    # Native ingester only: one entry per `written_paths` entry, the backfill
+    # cursor of the page that file came from. Phase 3 resumes the next chunk
+    # there when it leaves files unimported at its deadline. Empty on gallery-dl.
+    written_cursors: list[str | None] = field(default_factory=list)
     stdout: str = ""
     stderr: str = ""
     return_code: int = 0
@@ -194,8 +198,10 @@ class DownloadResult:
     # but not yet imported is invisible to every later walk, so a run killed in
     # between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13
     # images to a stranded run, 2026-09-24). Unmarked, the next walk finds the
-    # file on disk with no ImageRecord and imports it. None on gallery-dl.
-    mark_seen_after_import: Callable[[], None] | None = None
+    # file on disk with no ImageRecord and imports it. Takes `skip_paths`: the
+    # written paths phase 3 did not import, which stay unmarked for the same
+    # reason. None on gallery-dl.
+    mark_seen_after_import: Callable[..., None] | None = None
 
 
 def extract_errors_warnings(stderr: str) -> str:

backend/app/services/importer.py

@@ -16,6 +16,7 @@ from dataclasses import dataclass, field
 from enum import StrEnum
 from pathlib import Path
 
+from celery.exceptions import SoftTimeLimitExceeded
 from PIL import Image
 from sqlalchemy import func, select, update
 from sqlalchemy.exc import IntegrityError
@@ -43,6 +44,7 @@ from ..utils.paths import (
 from ..utils.phash import compute_phash, find_similar, fingerprint_path, fingerprints_match
 from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
+from ..utils.staging import is_temp_staging_file
 from .archive_extractor import extract_archive, is_archive
 from .attachment_store import AttachmentStore
 from .audits import single_color
@@ -115,6 +117,21 @@ def is_video(path: Path) -> bool:
     return path.suffix.lower() in VIDEO_EXTS
 
 
+def _not_content(path: Path) -> ImportResult | None:
+    """The skip result for a file that is never library content: a sidecar
+    `.json` (metadata) or a downloader's staging file (an unfinished download,
+    which becomes content only under its final name). None for anything else."""
+    if path.suffix.lower() == ".json":
+        error = "sidecar json is metadata, not content"
+    elif is_temp_staging_file(path):
+        error = "unfinished download (staging file), not content"
+    else:
+        return None
+    return ImportResult(
+        status="skipped", skip_reason=SkipReason.invalid_image, error=error,
+    )
+
+
 def _safe_ext(path: Path) -> str:
     """Conservatively extract a file extension for PostAttachment.ext
     (varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for
@@ -291,6 +308,44 @@ class Importer:
             sp.rollback()
             return self.session.execute(stmt).scalar_one()
 
+    def _store_new_record(self, record: ImageRecord) -> ImageRecord | None:
+        """Insert `record`, or return the row that already holds its bytes.
+
+        The sha256 lookup before an insert can lose a race: two copies of one
+        picture in different folders, imported side by side by two workers,
+        both find no row, and the unique index refuses the second insert. Only
+        that insert is undone (a savepoint), so the session stays usable and
+        the caller files its file as a duplicate of the winner. None means
+        `record` was stored."""
+        sp = self.session.begin_nested()
+        try:
+            self.session.add(record)
+            self.session.flush()
+            sp.commit()
+            return None
+        except IntegrityError:
+            sp.rollback()
+            winner = self.session.execute(
+                select(ImageRecord).where(ImageRecord.sha256 == record.sha256)
+            ).scalar_one_or_none()
+            if winner is None:
+                raise
+            return winner
+
+    def _file_as_duplicate(
+        self, existing: ImageRecord, path: Path, artist: Artist | None,
+        explicit_source: Source | None,
+    ) -> ImportResult:
+        """A byte-identical copy of `existing`: link the copy's post to the
+        image (enrich-on-duplicate), so a picture posted twice shows on both
+        posts, and report the skip."""
+        self._apply_sidecar(existing, path, artist, explicit_source=explicit_source)
+        self.session.commit()
+        return ImportResult(
+            status="skipped", skip_reason=SkipReason.duplicate_hash,
+            image_id=existing.id, error="sha256 already present",
+        )
+
     def _find_or_create_source(
         self, *, artist_id: int, platform: str, url: str,
     ) -> Source:
@@ -410,11 +465,8 @@ class Importer:
         media members (one Post via the archive-adjacent sidecar) and
         preserve the archive itself. Other non-media → PostAttachment.
         Nothing is skipped for being non-media (FC-2d-iii)."""
-        if source.suffix.lower() == ".json":
-            return ImportResult(
-                status="skipped", skip_reason=SkipReason.invalid_image,
-                error="sidecar json is metadata, not content",
-            )
+        if (refused := _not_content(source)) is not None:
+            return refused
         if is_archive(source):
             return self._import_archive(source)
         if not is_supported(source):
@@ -564,6 +616,9 @@ class Importer:
 
         artist_use = artist if artist is not None else self._resolve_artist(source)
         post = self._post_for_sidecar(source, artist_use, source_row=source_row)
+        # Committed before extraction, so a member that fails (and is rolled
+        # back on its own) can never take the archive's post with it.
+        self.session.commit()
         member_ids: list[int] = []
         # Every member image touched (new + superseded + deduped), so the
         # from_attachment_id stamp below covers files that already existed in the
@@ -690,7 +745,15 @@ class Importer:
                             member_record_ids.add(res.image_id)
                     else:
                         counts["failed"] += 1
+        except SoftTimeLimitExceeded:
+            # The task's time budget is spent: stop here so the task marks
+            # its row failed, rather than carrying on until the hard kill.
+            raise
         except Exception as exc:  # noqa: BLE001 — defensive per level; keep going
+            # Each member commits on its own, so this undoes only the one in
+            # progress; without it the next query fails on the dead transaction
+            # and the archive is never stored.
+            self.session.rollback()
             log.warning(
                 "archive extraction failed for %s (depth %d): %s",
                 archive_path.name, depth, exc,
@@ -871,18 +934,8 @@ class Importer:
         existing = self.session.execute(existing_stmt).scalar_one_or_none()
         if existing:
             if not self.deep:
-                # Enrich-on-duplicate (parity with attach_in_place): a re-scanned
-                # file that is a byte-dup of an existing image still links its
-                # post via _apply_sidecar, so a cross-posted image shows on every
-                # post.
-                self._apply_sidecar(
-                    existing, attribution_path, path_artist,
-                    explicit_source=explicit_source,
-                )
-                self.session.commit()
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.duplicate_hash,
-                    image_id=existing.id, error="sha256 already present",
+                return self._file_as_duplicate(
+                    existing, attribution_path, path_artist, explicit_source,
                 )
             return self._deep_rederive(existing, source, attribution_path)
 
@@ -968,8 +1021,14 @@ class Importer:
             origin="imported_filesystem",
             integrity_status="unknown",
         )
-        self.session.add(record)
-        self.session.flush()
+        if (winner := self._store_new_record(record)) is not None:
+            # The copy just written is a stray unless it is the winner's own
+            # file (the same source imported twice lands on the same path).
+            if Path(winner.path) != dest:
+                dest.unlink(missing_ok=True)
+            return self._file_as_duplicate(
+                winner, attribution_path, path_artist, explicit_source,
+            )
         self._phash_cache_append(phash, width, height, record.id)
 
         # Folder→artist (path_artist resolved up-front above); bind it to the
@@ -1219,11 +1278,8 @@ class Importer:
             caller may delete the on-disk original (mirrors duplicate_hash)
           - failed → file untouched; caller decides
         """
-        if path.suffix.lower() == ".json":
-            return ImportResult(
-                status="skipped", skip_reason=SkipReason.invalid_image,
-                error="sidecar json is metadata, not content",
-            )
+        if (refused := _not_content(path)) is not None:
+            return refused
         if is_archive(path):
             return self._import_archive(
                 path, artist=artist, source_row=source,
@@ -1300,12 +1356,7 @@ class Importer:
             # only sets it when NULL. Without this the new post is left with no
             # image AND (if its other content also deduped) no attachment, i.e. a
             # bare shell — operator-flagged 2026-06-08 (1589 empty Anduo posts).
-            self._apply_sidecar(existing, path, artist, explicit_source=source)
-            self.session.commit()
-            return ImportResult(
-                status="skipped", skip_reason=SkipReason.duplicate_hash,
-                image_id=existing.id, error="sha256 already present",
-            )
+            return self._file_as_duplicate(existing, path, artist, source)
 
         # phash dedup + supersede
         phash = None
@@ -1385,8 +1436,8 @@ class Importer:
         )
         if artist is not None:
             record.artist_id = artist.id
-        self.session.add(record)
-        self.session.flush()
+        if (winner := self._store_new_record(record)) is not None:
+            return self._file_as_duplicate(winner, path, artist, source)
         self._phash_cache_append(phash, width, height, record.id)
 
         # Sidecar provenance (best-effort). When `source` is passed, link
@@ -1401,6 +1452,51 @@ class Importer:
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
+    def discard_after_attach(self, path: Path, result: ImportResult) -> bool:
+        """Delete a downloaded file that `attach_in_place` has finished with,
+        when keeping it would only leave a second copy or a rejected file on
+        disk. Returns True when the file was removed. Both download paths (a
+        subscription walk's import step and the external-link worker) apply
+        this one policy:
+
+        - attached: the attachments store holds the canonical copy.
+        - imported or superseded ARCHIVE: its new images are in the library and
+          the archive itself is in the attachments store.
+        - duplicate skip: the library already holds the content.
+        - soft skip (too small, too transparent, single colour, not a readable
+          image): the import filters rejected it. A file left on disk with no
+          record is picked up by the next walk and imported again; the seen
+          ledger is what stops it being downloaded again.
+        - failed: the archive probe crashed, and a retry would crash again.
+
+        An imported or superseded media file is kept: it IS the record's file.
+        So is any path an ImageRecord points at, whatever the status, because a
+        re-fetch can write identical bytes over a record's own file. A staging
+        file is kept too: it may be a download still being written, and the
+        temp-file sweep removes it once it is old enough to be abandoned."""
+        if is_temp_staging_file(path):
+            return False
+        if result.status in ("imported", "superseded"):
+            redundant = is_archive(path)
+        else:
+            redundant = result.status in ("attached", "skipped", "failed")
+        if not redundant:
+            return False
+        owned = self.session.execute(
+            select(ImageRecord.id).where(
+                ImageRecord.path.in_({str(path), str(path.resolve())})
+            ).limit(1)
+        ).first()
+        if owned is not None:
+            log.info("keeping %s: it is image %s's file", path.name, owned.id)
+            return False
+        try:
+            path.unlink(missing_ok=True)
+        except OSError as exc:
+            log.warning("could not remove %s: %s", path, exc)
+            return False
+        return True
+
     def _upsert_artist(self, name: str) -> Artist:
         slug = slugify(name)
         artist = self.session.execute(

backend/app/services/ingest_core.py

@@ -114,13 +114,29 @@ _CANARY_MIN_SAMPLE = 30
 # on-disk files, and died at the soft limit.
 #
 # So the walk also stops when its elapsed time PLUS phase 3's estimated cost
-# would pass CHUNK_TOTAL_SECONDS. Costs measured on the live instance: 431
-# imports took 976s (~2.3s each: hash, pHash, sidecar, provenance); a relink is
-# a sha256 over NFS, 0.15s for the 8.8 MB average file, plus a lookup.
-# test_download_source_task pins CHUNK_TOTAL_SECONDS under the soft limit.
+# would pass CHUNK_TOTAL_SECONDS. An import (hash, pHash, sidecar, provenance,
+# thumbnail enqueue) runs 2-5s depending on the files; the estimate takes the
+# slow end. A relink is a sha256 over NFS plus a lookup.
+#
+# The estimate is a plan, not a guard. PHASE3_DEADLINE_SECONDS is the guard:
+# phase 3 starts no import past it (download_service), and leaves the rest on
+# disk, unmarked, for the next chunk. It sits under the soft limit by enough to
+# finish the import in hand, upsert post records and finalize the event.
+# test_download_source_task pins both under the soft limit.
 CHUNK_TOTAL_SECONDS = 1200.0
-PHASE3_IMPORT_SECONDS = 2.5
+PHASE3_IMPORT_SECONDS = 5.0
 PHASE3_RELINK_SECONDS = 0.25
+PHASE3_DEADLINE_SECONDS = 1200.0
+
+
+def _unskipped(
+    fetched: list[tuple[str, str]],
+    written_keys: dict[str, tuple[str, str]],
+    skip_paths,
+) -> list[tuple[str, str]]:
+    """`fetched` without the ledger keys of files phase 3 left unimported."""
+    skip = {written_keys[p] for p in skip_paths if p in written_keys}
+    return [kv for kv in fetched if kv not in skip] if skip else list(fetched)
 
 
 def _parse_published(raw: object) -> datetime | None:
@@ -289,6 +305,12 @@ class Ingester:
         # Media handed to phase 3 for import. Marked seen by phase 3 once the
         # import has run (`mark_seen_after_import`), not here — see there.
         fetched: list[tuple[str, str]] = []
+        # Per entry of `written`: the cursor of the page it came from, and its
+        # ledger key. Phase 3 can stop importing at its deadline; the files it
+        # leaves must stay unmarked, and the next chunk has to resume on the
+        # page holding the first of them, or no walk sees them again.
+        written_cursors: list[str | None] = []
+        written_keys: dict[str, tuple[str, str]] = {}
         # Post-record keys written this walk. Marked with the media, after
         # phase 3 has upserted the records — marking them at write time left a
         # walk killed before phase 3 with posts the ledger calls recorded that
@@ -347,8 +369,10 @@ class Ingester:
                 written_paths=written,
                 post_record_paths=list(post_records),
                 relink_source_paths=list(relink),
-                mark_seen_after_import=lambda: self._mark_seen(
-                    source_id, fetched + recorded,
+                written_cursors=list(written_cursors),
+                mark_seen_after_import=lambda skip_paths=(): self._mark_seen(
+                    source_id,
+                    _unskipped(fetched, written_keys, skip_paths) + recorded,
                 ),
                 stdout="\n".join(log_lines),
                 stderr="",
@@ -577,6 +601,8 @@ class Ingester:
                         downloaded += 1
                         if outcome.path is not None:
                             written.append(str(outcome.path))
+                            written_cursors.append(emitted_cursor)
+                            written_keys[str(outcome.path)] = (key, media_item.post_id)
                         fetched.append((key, media_item.post_id))
                         to_clear.append(key)
                         consecutive_seen = 0
@@ -590,6 +616,8 @@ class Ingester:
                         # attach_in_place because no record owns this path —
                         # the unlink below is about a file that IS the record.
                         written.append(str(outcome.path))
+                        written_cursors.append(emitted_cursor)
+                        written_keys[str(outcome.path)] = (key, media_item.post_id)
                         fetched.append((key, media_item.post_id))
                         to_clear.append(key)
                         skipped_count += 1

backend/app/services/library_scan.py

@@ -0,0 +1,181 @@
+"""Files in the artists' folders that the library has no record of.
+
+A download writes its files into the artist's folder and imports them in a
+later step. A run that stops between the two (a restart, a time limit) leaves
+complete files on disk that no image or attachment points at, and a walk that
+has already marked them seen never comes back for them. `find_unimported`
+lists those files; `queue_unimported` turns the same list into an import batch
+that attaches each file where it lies (`scan_mode="library"`), so a file the
+library already holds, or one the import filters reject, is removed instead.
+
+Only files under a folder named for an existing artist (matched by slug, so a
+differently-cased folder still matches) are considered, and only images,
+videos and archives. Files changed in the last day are left for the download
+that may still be writing them.
+"""
+from __future__ import annotations
+
+import os
+import stat
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..models import Artist, ImageRecord, ImportBatch, ImportTask, PostAttachment, Source
+from ..utils.slug import slugify
+from ..utils.staging import is_temp_staging_file
+from .archive_extractor import is_archive
+from .importer import is_supported
+
+# Top-level folders that hold the library's own stores, not an artist's files.
+_STORE_DIRS = frozenset({"thumbs", "attachments", "cookies", "secrets"})
+MIN_AGE_SECONDS = 24 * 3600
+_ACTIVE_TASK_STATUSES = ("pending", "queued", "processing")
+_SAMPLES_PER_ARTIST = 3
+_ARTISTS_LISTED = 50
+
+
+@dataclass(frozen=True)
+class Unimported:
+    path: Path
+    artist_slug: str
+    task_type: str  # "media" or "archive", as ImportTask.task_type
+    size: int
+
+
+def _referenced_paths(session: Session) -> set[str]:
+    paths: set[str] = set()
+    for p, thumb in session.execute(select(ImageRecord.path, ImageRecord.thumbnail_path)):
+        paths.add(p)
+        if thumb:
+            paths.add(thumb)
+    paths.update(session.execute(select(PostAttachment.path)).scalars())
+    paths.update(session.execute(
+        select(ImportTask.source_path).where(ImportTask.status.in_(_ACTIVE_TASK_STATUSES))
+    ).scalars())
+    return paths
+
+
+def find_unimported(
+    session: Session, images_root: Path, *, now: float | None = None,
+) -> tuple[list[Unimported], dict]:
+    """Walk the artists' folders and return (files to import, summary).
+
+    The summary counts what was found and what was passed over, and lists the
+    artists with the most to import (bytes first) with a few sample paths each:
+        {"files", "bytes", "archives", "artists": [{slug, files, bytes,
+         samples}], "recent", "not_media", "no_artist"}"""
+    now = time.time() if now is None else now
+    referenced = _referenced_paths(session)
+    slugs = set(session.execute(select(Artist.slug)).scalars())
+    found: list[Unimported] = []
+    passed = {"recent": 0, "not_media": 0, "no_artist": 0}
+    root = Path(images_root)
+    for top in sorted(os.listdir(root)):
+        if top in _STORE_DIRS or top.startswith(("_", ".")):
+            continue
+        top_dir = root / top
+        if not top_dir.is_dir():
+            continue
+        slug = slugify(top)
+        known = slug in slugs
+        for dirpath, dirnames, filenames in os.walk(top_dir):
+            dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
+            for name in sorted(filenames):
+                path = Path(dirpath) / name
+                if (
+                    name.startswith(".") or path.suffix.lower() == ".json"
+                    or is_temp_staging_file(path)
+                ):
+                    continue
+                if str(path) in referenced:
+                    continue
+                try:
+                    st = path.lstat()
+                except OSError:
+                    continue
+                if not stat.S_ISREG(st.st_mode):
+                    continue
+                kind = "archive" if is_archive(path) else ("media" if is_supported(path) else None)
+                if kind is None:
+                    passed["not_media"] += 1
+                elif now - st.st_mtime < MIN_AGE_SECONDS:
+                    passed["recent"] += 1
+                elif not known:
+                    passed["no_artist"] += 1
+                else:
+                    found.append(Unimported(path, slug, kind, st.st_size))
+
+    per_artist: dict[str, dict] = defaultdict(lambda: {"files": 0, "bytes": 0, "samples": []})
+    for u in found:
+        a = per_artist[u.artist_slug]
+        a["files"] += 1
+        a["bytes"] += u.size
+        if len(a["samples"]) < _SAMPLES_PER_ARTIST:
+            a["samples"].append(str(u.path.relative_to(root)))
+    artists = sorted(
+        ({"slug": s, **v} for s, v in per_artist.items()), key=lambda a: -a["bytes"],
+    )[:_ARTISTS_LISTED]
+    summary = {
+        "files": len(found),
+        "bytes": sum(u.size for u in found),
+        "archives": sum(1 for u in found if u.task_type == "archive"),
+        "artists": artists,
+        **passed,
+    }
+    return found, summary
+
+
+def queue_unimported(
+    session: Session, images_root: Path, *, triggered_by: str = "manual",
+) -> dict:
+    """Create a `library` import batch holding every file `find_unimported`
+    lists, and queue it. Returns the same summary plus `batch_id` (None when
+    there was nothing to import)."""
+    from ..tasks.import_file import enqueue_import  # tasks import services
+
+    found, summary = find_unimported(session, images_root)
+    summary["batch_id"] = None
+    if not found:
+        return summary
+    batch = ImportBatch(
+        triggered_by=triggered_by, source_path=str(images_root),
+        scan_mode="library", status="running", total_files=len(found),
+    )
+    session.add(batch)
+    session.flush()
+    tasks = [
+        ImportTask(
+            batch_id=batch.id, source_path=str(u.path), task_type=u.task_type,
+            status="queued", size_bytes=u.size,
+        )
+        for u in found
+    ]
+    session.add_all(tasks)
+    session.commit()
+    for t in tasks:
+        enqueue_import(t.id, t.task_type)
+    summary["batch_id"] = batch.id
+    return summary
+
+
+def folder_owner(
+    session: Session, path: Path, images_root: Path,
+) -> tuple[Artist | None, Source | None]:
+    """The artist whose folder holds `path`, and that artist's source for the
+    platform folder below it when there is exactly one (a Discord artist has a
+    source per channel, so there the post's sidecar decides)."""
+    parts = Path(path).relative_to(images_root).parts
+    artist = session.execute(
+        select(Artist).where(Artist.slug == slugify(parts[0]))
+    ).scalar_one_or_none()
+    if artist is None or len(parts) < 3:
+        return artist, None
+    sources = session.execute(
+        select(Source).where(Source.artist_id == artist.id, Source.platform == parts[1])
+    ).scalars().all()
+    return artist, (sources[0] if len(sources) == 1 else None)

backend/app/services/post_feed_service.py

@@ -479,6 +479,10 @@ class PostFeedService:
             # the UI uses this to explain why a post it linked to is not in the
             # stream.
             "absorbed_by_post_id": post.absorbed_by_post_id,
+            # #4481. The Discord channel a message was posted in: both the native
+            # post record and gallery-dl's sidecars carry it as `channel`. None
+            # on every other platform, and on a Discord post that never said.
+            "channel": _discord_channel(post, source),
             "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
             "source": (
                 {"id": source.id, "platform": source.platform}
@@ -488,3 +492,13 @@ class PostFeedService:
             "thumbnails_more": thumbs_entry["more"],
             "attachments": atts_map.get(post.id, []),
         }
+
+
+def _discord_channel(post: Post, source: Source | None) -> str | None:
+    if source is None or source.platform != "discord":
+        return None
+    raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {}
+    channel = raw.get("channel")
+    if not isinstance(channel, str):
+        return None
+    return channel.strip() or None

backend/app/services/source_service.py

@@ -68,6 +68,9 @@ class SourceRecord:
     artist_slug: str
     platform: str
     url: str
+    # alembic 0115: the name the platform gives it, where the URL is opaque
+    # (a Discord link is two ids). None until a walk has read it.
+    display_name: str | None
     enabled: bool
     config_overrides: dict | None
     last_checked_at: str | None
@@ -110,6 +113,7 @@ class SourceRecord:
             "artist_slug": self.artist_slug,
             "platform": self.platform,
             "url": self.url,
+            "display_name": self.display_name,
             "enabled": self.enabled,
             "config_overrides": self.config_overrides,
             "last_checked_at": self.last_checked_at,
@@ -301,6 +305,7 @@ class SourceService:
             artist_slug=artist.slug,
             platform=source.platform,
             url=source.url,
+            display_name=source.display_name,
             enabled=source.enabled,
             config_overrides=source.config_overrides,
             last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,

backend/app/tasks/admin.py

@@ -459,3 +459,29 @@ def reclaim_orphaned_attachments_task(self, dry_run: bool = True) -> dict:
         return cleanup_service.reclaim_orphaned_attachments(
             session, images_root=IMAGES_ROOT, dry_run=dry_run,
         )
+
+
+@celery.task(
+    name="backend.app.tasks.admin.import_unimported_files_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    # One walk of the artists' folders: a stat per file, so minutes on a large
+    # library on network storage. The limits are the backstop for a wedged
+    # filesystem, not the expected exit.
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def import_unimported_files_task(self, dry_run: bool = True) -> dict:
+    """Find files in the artists' folders that the library has no record of.
+    dry_run (the default) returns what was found, per artist, without changing
+    anything; apply queues them as a `library` import batch, which attaches each
+    file where it lies and removes the ones the library already holds or the
+    import filters reject."""
+    from ..services import library_scan
+
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        if dry_run:
+            _found, summary = library_scan.find_unimported(session, IMAGES_ROOT)
+            return {**summary, "batch_id": None}
+        return library_scan.queue_unimported(session, IMAGES_ROOT)

backend/app/tasks/backup.py

@@ -8,6 +8,7 @@ BackupRun.
 from __future__ import annotations
 
 import logging
+import time
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -18,7 +19,7 @@ from sqlalchemy.exc import DBAPIError, OperationalError
 from ..celery_app import celery
 from ..config import get_config
 from ..models import BackupRun, ImportSettings
-from ..services import backup_service
+from ..services import backup_destination, backup_media, backup_s3, backup_service
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
 log = logging.getLogger(__name__)
@@ -49,7 +50,12 @@ def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
 )
 def backup_db_task(self, *, tag: str | None = None,
                    triggered_by: str = "manual") -> dict:
-    """Create one DB backup. Returns {'backup_run_id': N}."""
+    """Create one DB backup. Returns {'backup_run_id': N}.
+
+    With an off-site destination set up, the dump streams to that bucket and
+    nothing is written here; the row's manifest says where it went
+    (`offsite`: endpoint, bucket, key). Without one, the dump is written to
+    /images/_backups."""
     SessionLocal = _sync_session_factory()
     cfg = get_config()
     now = datetime.now(UTC)
@@ -64,10 +70,19 @@ def backup_db_task(self, *, tag: str | None = None,
         run_id = row.id
 
     try:
-        result = backup_service.backup_db(
-            db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
-            tag=tag, triggered_by=triggered_by,
-        )
+        with SessionLocal() as session:
+            settings = ImportSettings.load_sync(session)
+            target = backup_destination.saved_target(settings)
+            limit_mbps = settings.backup_upload_limit_mbps
+        if target is not None:
+            result = backup_service.backup_db_offsite(
+                db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps,
+            )
+        else:
+            result = backup_service.backup_db(
+                db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
+                tag=tag, triggered_by=triggered_by,
+            )
     except (SoftTimeLimitExceeded, Exception) as exc:
         with SessionLocal() as session:
             row = session.get(BackupRun, run_id)
@@ -79,12 +94,23 @@ def backup_db_task(self, *, tag: str | None = None,
         row = session.get(BackupRun, run_id)
         row.status = "ok"
         row.finished_at = datetime.now(UTC)
-        row.sql_path = result["sql_path"]
         row.size_bytes = result["size_bytes"]
-        row.manifest = {
-            "manifest_path": result["manifest_path"],
-            "ts": result["ts"],
-        }
+        if target is not None:
+            row.manifest = {
+                "ts": result["ts"],
+                "offsite": {
+                    "endpoint": result["endpoint"],
+                    "bucket": result["bucket"],
+                    "key": result["key"],
+                    "parts": result["parts"],
+                },
+            }
+        else:
+            row.sql_path = result["sql_path"]
+            row.manifest = {
+                "manifest_path": result["manifest_path"],
+                "ts": result["ts"],
+            }
         session.commit()
     return {"backup_run_id": run_id}
 
@@ -242,7 +268,9 @@ def restore_images_task(self, *, source_backup_run_id: int) -> dict:
 def prune_backups() -> dict:
     """Daily Beat. Per-kind retention from ImportSettings.
 
-    Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
+    Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
+    "files_unlinked": K}. Media runs leave no files here; only their history
+    is trimmed, to the latest MEDIA_RUNS_KEPT.
     Tagged rows (tag IS NOT NULL) are never pruned.
     Status='running' / 'restoring' rows are never pruned — the
     recover_stalled_backup_runs sweep flips truly-stuck ones to
@@ -251,12 +279,13 @@ def prune_backups() -> dict:
     Audit 2026-06-02 added the dedicated sweep.)
     """
     SessionLocal = _sync_session_factory()
-    counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
+    counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0, "files_unlinked": 0}
     with SessionLocal() as session:
         s = ImportSettings.load_sync(session)
         for kind, keep in (
             ("db", s.backup_db_keep_last_n),
             ("images", s.backup_images_keep_last_n),
+            ("media", MEDIA_RUNS_KEPT),
         ):
             candidates = session.execute(
                 select(BackupRun)
@@ -300,4 +329,168 @@ def backup_db_nightly() -> dict:
     if now_hour != configured_hour:
         return {"skipped": f"hour={now_hour} != configured={configured_hour}"}
     res = backup_db_task.delay(triggered_by="nightly")
-    return {"dispatched": res.id}
+    out = {"dispatched": res.id}
+    # With an off-site destination, the night's backup is the whole library:
+    # the database dump, then any images and attachments not yet in the bucket.
+    # Both run on the long-maintenance lane, one after the other.
+    with SessionLocal() as session:
+        try:
+            offsite = backup_destination.saved_target(ImportSettings.load_sync(session))
+        except Exception:  # noqa: BLE001 — the media run reports it itself
+            offsite = True
+    if offsite is not None:
+        out["media_dispatched"] = backup_media_task.delay(triggered_by="nightly").id
+    return out
+
+
+# ── Off-site copy of the library's files ─────────────────────────────────────
+
+# Each task invocation sends files for this long, then queues the next chunk
+# of the same run, so other long-maintenance jobs get their turn in between.
+MEDIA_CHUNK_SECONDS = 45 * 60
+MEDIA_RUNS_KEPT = 60
+# A failed file is listed on the run, up to this many; the count covers all.
+MEDIA_FAILURES_LISTED = 20
+
+
+def _offsite_manifest(target) -> dict:
+    return {"endpoint": target.endpoint, "bucket": target.bucket,
+            "key": backup_media.PREFIX}
+
+
+@celery.task(
+    name="backend.app.tasks.backup.backup_media_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=30, retry_backoff_max=300, max_retries=2,
+    # A chunk stops starting files after MEDIA_CHUNK_SECONDS; the limits leave
+    # room for the file in flight, a multi-gigabyte archive at a low upload
+    # limit included.
+    soft_time_limit=21600, time_limit=23400,
+)
+def backup_media_task(self, *, run_id: int | None = None,
+                      triggered_by: str = "manual") -> dict:
+    """Send the library's images and attachments that aren't in the off-site
+    bucket yet, one chunk of a run. Called without `run_id` it starts a run
+    (unless one is already going); each chunk queues the next until nothing is
+    left. A file that can't be sent is listed on the run and tried again by
+    the next run. Returns {'backup_run_id': N, ...}."""
+    SessionLocal = _sync_session_factory()
+    now = datetime.now(UTC)
+    with SessionLocal() as session:
+        if run_id is None:
+            going = session.execute(
+                select(BackupRun.id).where(BackupRun.kind == "media")
+                .where(BackupRun.status == "running").limit(1)
+            ).scalar_one_or_none()
+            if going is not None:
+                return {"backup_run_id": going, "skipped": "a media backup is already running"}
+            row = BackupRun(kind="media", status="running", triggered_by=triggered_by,
+                            started_at=now, last_progress_at=now, size_bytes=0,
+                            manifest={"sent": 0, "failed": 0, "failures": [], "cursor": ""})
+            session.add(row)
+            session.commit()
+            run_id = row.id
+        else:
+            row = session.get(BackupRun, run_id)
+            if row is None or row.status != "running":
+                return {"backup_run_id": run_id, "skipped": "run is no longer running"}
+
+    try:
+        with SessionLocal() as session:
+            settings = ImportSettings.load_sync(session)
+            target = backup_destination.saved_target(settings)
+            limit_mbps = settings.backup_upload_limit_mbps
+        if target is None:
+            raise backup_s3.S3Error("No off-site destination is set up.")
+        more = _send_media_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps))
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            if row is not None:
+                _mark_failed(session, row, exc)
+        raise
+
+    if more:
+        backup_media_task.delay(run_id=run_id)
+        return {"backup_run_id": run_id, "continues": True}
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        row.status = "ok"
+        row.finished_at = datetime.now(UTC)
+        if m.get("failed"):
+            first = (m.get("failures") or [{}])[0]
+            row.error = (f"{m['failed']} file(s) weren't sent; the next run tries them again. "
+                         f"First: {first.get('path')}: {first.get('error')}")[:2000]
+        session.commit()
+    return {"backup_run_id": run_id, "continues": False}
+
+
+def _send_media_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
+    """Send files until the chunk's time is up or none are left. Progress is
+    committed after every file, so a chunk that dies loses at most the file
+    in flight. Returns True when files remain for another chunk."""
+    tid = backup_media.target_id(target)
+    deadline = time.monotonic() + MEDIA_CHUNK_SECONDS
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        m["offsite"] = _offsite_manifest(target)
+        cursor = m.get("cursor") or ""
+        while True:
+            batch = backup_media.pending(session, tid, after=cursor)
+            # End the read before any upload starts: a transaction left open
+            # across a long transfer holds back vacuum for its whole length.
+            session.commit()
+            if not batch:
+                m["cursor"] = cursor
+                row.manifest = dict(m)
+                row.last_progress_at = datetime.now(UTC)
+                session.commit()
+                return False
+            for blob in batch:
+                if time.monotonic() > deadline:
+                    m["cursor"] = cursor
+                    row.manifest = dict(m)
+                    session.commit()
+                    return True
+                try:
+                    key = backup_media.send(target, blob, IMAGES_ROOT, pacer)
+                except (backup_s3.S3Error, OSError) as exc:
+                    m["failed"] = m.get("failed", 0) + 1
+                    if len(m.get("failures") or []) < MEDIA_FAILURES_LISTED:
+                        m["failures"] = [*(m.get("failures") or []),
+                                         {"path": blob.path, "error": str(exc)[:300]}]
+                else:
+                    backup_media.record(session, tid, [(blob, key)])
+                    m["sent"] = m.get("sent", 0) + 1
+                    row.size_bytes = (row.size_bytes or 0) + blob.size_bytes
+                cursor = blob.sha256
+                m["cursor"] = cursor
+                row.manifest = dict(m)
+                row.last_progress_at = datetime.now(UTC)
+                session.commit()
+
+
+@celery.task(
+    name="backend.app.tasks.backup.reconcile_media_backup",
+    soft_time_limit=1800, time_limit=2100,
+)
+def reconcile_media_backup() -> dict:
+    """Weekly Beat. Compare what the off-site bucket holds under `media/` with
+    what FabledCurator believes it sent: a missing object is sent again by
+    the next run, and an object already there is not sent twice. Skipped
+    while a media run is going, and when no destination is set up."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        target = backup_destination.saved_target(ImportSettings.load_sync(session))
+        if target is None:
+            return {"skipped": "no destination"}
+        going = session.execute(
+            select(BackupRun.id).where(BackupRun.kind == "media")
+            .where(BackupRun.status == "running").limit(1)
+        ).scalar_one_or_none()
+        if going is not None:
+            return {"skipped": "a media backup is running"}
+        return backup_media.reconcile(session, target)

backend/app/tasks/external.py

@@ -24,7 +24,7 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 import redis
-from sqlalchemy import delete, select, update
+from sqlalchemy import delete, or_, select, update
 
 from ..celery_app import celery
 from ..config import get_config
@@ -32,6 +32,7 @@ from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, So
 from ..services.external_fetch import fetch_external
 from ..services.importer import Importer
 from ..services.thumbnailer import Thumbnailer
+from ..utils.staging import is_temp_staging_file
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
 log = logging.getLogger(__name__)
@@ -39,14 +40,20 @@ log = logging.getLogger(__name__)
 IMAGES_ROOT = Path("/images")
 
 # After this many failed attempts a link is dead-lettered (skipped by routine
-# sweeps; an operator recovery still re-attempts). Mirrors the ingester ledger.
+# sweeps; the daily recovery retries it, sparingly). Mirrors the ingester ledger.
 DEAD_LETTER_THRESHOLD = 3
 # Per-fetch read + total budgets now live in external_fetch (a short read
 # timeout fails a stalled host fast; a generous total caps a big-but-flowing
 # download). The celery soft/hard time_limit is the outer backstop above those.
 # Links enqueued per sweep — bounds the burst when a big backfill records many.
 _SWEEP_BATCH = 50
-# Dead rows older than this are pruned (retention).
+# Recovery gives a dead link one more attempt once it has been dead for
+# 2**recoveries days (1, 2, 4, 8, 16), and stops after _MAX_RECOVERIES. A host
+# outage gets several later chances; a link that can never succeed (too big to
+# import in time, a file made private) costs a handful of downloads, not three
+# a day.
+_MAX_RECOVERIES = 5
+# A link recovery has given up on is pruned this long after its last attempt.
 _RETENTION_DAYS = 30
 # Per-host serialize lock. TTL is a safety net for a worker that dies holding it
 # (normal completion/error releases it in `finally`); sized just past the fetch
@@ -54,6 +61,9 @@ _RETENTION_DAYS = 30
 # than one fetch would have taken.
 _LOCK_PREFIX = "fc:extdl_lock:"
 _LOCK_TTL = 2400
+# A link 'downloading' this long has outlived fetch_external_link's hard limit
+# (time_limit=3600), so its worker is gone and the sweep recovers it.
+_STALL_MINUTES = 65
 _SERIALIZE_COUNTDOWN = 120
 _MAX_SERIALIZE_WAITS = 30
 
@@ -106,7 +116,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
                 ExternalLink.id == link_id,
                 ExternalLink.status.in_(("pending", "failed")),
             )
-            .values(status="downloading")
+            .values(status="downloading", started_at=datetime.now(UTC))
             .returning(ExternalLink.id)
         ).first()
         session.commit()
@@ -180,6 +190,9 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
         / str(post.external_post_id) / str(link_id)
     )
     try:
+        # Clear the staging files an interrupted attempt left behind. Complete
+        # files it left stay, and are imported once this fetch succeeds.
+        _discard_partials(post_dir)
         result = fetch_external(host, url, post_dir)
         with SessionLocal() as session:
             link = session.get(ExternalLink, link_id)
@@ -190,6 +203,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
                 )
                 _record_failure(link, attempts, result.error or "fetch failed")
                 session.commit()
+                _discard_partials(post_dir)
                 return {"link_id": link_id, "error": result.error}
 
             log.info(
@@ -201,7 +215,8 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             post = session.get(Post, link.post_id)
             artist = session.get(Artist, post.artist_id)
             source = session.get(Source, post.source_id) if post.source_id else None
-            image_ids = _route_files(session, result.files, post, platform, artist, source)
+            files = _unimported_files(session, post_dir, first=result.files)
+            image_ids = _route_files(session, files, post, platform, artist, source)
             link.status = "downloaded"
             link.last_error = None
             link.completed_at = datetime.now(UTC)
@@ -231,6 +246,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             if link is not None and link.status == "downloading":
                 _record_failure(link, attempts, f"{type(exc).__name__}: {exc}")
                 session.commit()
+        _discard_partials(post_dir)
         raise
     finally:
         if lock is not None:
@@ -287,10 +303,9 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
                 post.id, f.name, result.status, len(ids),
             )
         elif result.status == "attached":
-            # Non-art / archive captured as a PostAttachment (copied into the
-            # store) — drop the on-disk original. An archive captured WITHOUT
-            # extracting any image carries the reason (the recurring "zip but no
-            # images" symptom, #718) — surface it loudly.
+            # Non-art, or an archive with no new images, captured as a
+            # PostAttachment. An archive captured WITHOUT extracting any image
+            # carries the reason, so the "zip but no images" case is visible.
             if result.error:
                 log.warning(
                     "extdl archive captured UNEXTRACTED: post=%s file=%s reason=%s",
@@ -298,51 +313,104 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
                 )
             else:
                 log.info("extdl attachment captured: post=%s file=%s", post.id, f.name)
-            f.unlink(missing_ok=True)
         elif result.status == "skipped":
             reason = result.skip_reason.value if result.skip_reason else None
             log.info(
                 "extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason,
             )
-            if reason in ("duplicate_hash", "duplicate_phash"):
-                # Path-safe unlink: external files are imported IN PLACE, so `f`
-                # can BE the canonical record's backing file (same content
-                # re-fetched / a colliding name). Only delete `f` when it is NOT
-                # the existing record's file — otherwise we orphan the record and
-                # playback 404s. (#859)
-                canonical = None
-                if result.image_id is not None:
-                    rec = session.get(ImageRecord, result.image_id)
-                    if rec is not None:
-                        canonical = (IMAGES_ROOT / rec.path).resolve()
-                if canonical != f.resolve():
-                    f.unlink(missing_ok=True)  # a redundant copy — safe to drop
-                else:
-                    log.info(
-                        "extdl keep: %s IS record %s's canonical file — not unlinking",
-                        f.name, result.image_id,
-                    )
         elif result.status == "failed":
             log.warning(
                 "extdl import FAILED: post=%s file=%s error=%s",
                 post.id, f.name, result.error,
             )
-            f.unlink(missing_ok=True)
         else:  # 'refreshed' or any future status — work happened, no new images
             log.info(
                 "extdl import: post=%s file=%s status=%s", post.id, f.name, result.status,
             )
+        # Same policy as the subscription download path: drop the file unless
+        # the library needs it at this path. A file imported in place is kept.
+        importer.discard_after_attach(f, result)
         # The synthesized sidecar has done its job — don't litter the tree.
         Path(str(f) + ".json").unlink(missing_ok=True)
     return image_ids
 
 
+def _unimported_files(session, link_dir: Path, *, first=()) -> list[Path]:
+    """Every complete file under a link's dir, subfolders included, that no
+    ImageRecord points at: what this fetch wrote plus anything an earlier
+    attempt finished writing but never imported. `first` (the fetch's own
+    files) leads the list. Staging files and sidecars are left out."""
+    if not link_dir.is_dir():
+        return []
+    found = {
+        p for p in link_dir.rglob("*")
+        if p.is_file() and not is_temp_staging_file(p) and p.suffix.lower() != ".json"
+    }
+    ordered = [p for p in first if p in found] + sorted(found.difference(first))
+    recorded = set(session.execute(
+        select(ImageRecord.path).where(
+            ImageRecord.path.in_([str(p) for p in ordered])
+        )
+    ).scalars()) if ordered else set()
+    return [p for p in ordered if str(p) not in recorded]
+
+
+def _discard_partials(link_dir: Path) -> int:
+    """Delete the staging files under a link's dir (megatools' `.megatmp.*`, an
+    HTTP fetch's `.part`), then the dirs left empty. Every fetcher writes to a
+    staging name and renames on completion, so a staging file is a transfer
+    that never finished. Complete files stay: they may be the only copy of art
+    whose upstream has since gone. Returns the number of files removed."""
+    if not link_dir.is_dir():
+        return 0
+    removed = 0
+    for p in link_dir.rglob("*"):
+        if not (p.is_file() and is_temp_staging_file(p)):
+            continue
+        try:
+            p.unlink(missing_ok=True)
+            removed += 1
+        except OSError as exc:
+            log.warning("extdl could not remove %s: %s", p, exc)
+    for d in sorted((p for p in link_dir.rglob("*") if p.is_dir()), reverse=True):
+        try:
+            d.rmdir()
+        except OSError:
+            pass  # not empty: it holds a complete file
+    try:
+        link_dir.rmdir()
+    except OSError:
+        pass
+    if removed:
+        log.info("extdl removed %d leftover file(s) from %s", removed, link_dir)
+    return removed
+
+
 @celery.task(name="backend.app.tasks.external.sweep_external_links")
 def sweep_external_links() -> dict:
     """Enqueue a bounded batch of actionable links (pending, or failed below the
-    dead-letter threshold). Driven by the post-download hook and a beat tick."""
+    dead-letter threshold). Driven by the post-download hook and a beat tick.
+
+    First it recovers links whose worker died mid-fetch: a link 'downloading'
+    for longer than the task's hard limit has no worker left, so it counts as a
+    failed attempt and goes back to the retry path (dead-lettered at the
+    threshold like any other failure). The next attempt clears its staging
+    files and imports the complete ones."""
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
+        now = datetime.now(UTC)
+        stale = session.execute(
+            select(ExternalLink).where(
+                ExternalLink.status == "downloading",
+                or_(
+                    ExternalLink.started_at.is_(None),
+                    ExternalLink.started_at < now - timedelta(minutes=_STALL_MINUTES),
+                ),
+            )
+        ).scalars().all()
+        for link in stale:
+            _record_failure(link, link.attempts, "worker stopped mid-fetch")
+        session.commit()
         ids = session.execute(
             select(ExternalLink.id).where(
                 ExternalLink.status.in_(("pending", "failed")),
@@ -351,34 +419,53 @@ def sweep_external_links() -> dict:
         ).scalars().all()
     for link_id in ids:
         fetch_external_link.delay(link_id)
-    return {"enqueued": len(ids)}
+    return {"enqueued": len(ids), "recovered": len(stale)}
 
 
 @celery.task(name="backend.app.tasks.external.recover_external_links")
 def recover_external_links() -> dict:
-    """Recovery sweep (rule #89): reset dead links back to retryable so a stuck
-    host outage or a since-fixed bug gets another pass, then enqueue. Bounded."""
+    """Recovery sweep: give each dead link one more attempt once it
+    has been dead for 2**recoveries days, then enqueue. A recovered link that
+    fails again goes straight back to dead, and after _MAX_RECOVERIES it stays
+    dead until retention prunes it."""
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
-        session.execute(
-            update(ExternalLink).where(ExternalLink.status == "dead")
-            .values(status="failed", attempts=0, last_error=None)
-        )
+        now = datetime.now(UTC)
+        candidates = session.execute(
+            select(ExternalLink).where(
+                ExternalLink.status == "dead",
+                ExternalLink.recoveries < _MAX_RECOVERIES,
+            )
+        ).scalars().all()
+        recovered = 0
+        for link in candidates:
+            wait = timedelta(days=2 ** link.recoveries)
+            if link.completed_at is not None and link.completed_at > now - wait:
+                continue
+            link.status = "failed"
+            link.attempts = DEAD_LETTER_THRESHOLD - 1
+            link.recoveries += 1
+            recovered += 1
         session.commit()
-    return sweep_external_links()
+    return {**sweep_external_links(), "revived": recovered}
 
 
 @celery.task(name="backend.app.tasks.external.prune_external_links")
 def prune_external_links() -> dict:
-    """Retention (rule #89): delete long-dead links so the ledger doesn't grow
-    unbounded. Downloaded links are kept (they're the record of what we have)."""
+    """Retention: delete links recovery has given up on, once their
+    last attempt is _RETENTION_DAYS old, so the ledger doesn't grow unbounded.
+    Downloaded links are kept (they're the record of what we have)."""
     cutoff = datetime.now(UTC) - timedelta(days=_RETENTION_DAYS)
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
         res = session.execute(
             delete(ExternalLink).where(
                 ExternalLink.status == "dead",
-                ExternalLink.created_at < cutoff,
+                ExternalLink.recoveries >= _MAX_RECOVERIES,
+                or_(
+                    ExternalLink.completed_at.is_(None),
+                    ExternalLink.completed_at < cutoff,
+                ),
             )
         )
         session.commit()

backend/app/tasks/import_file.py

@@ -19,7 +19,8 @@ from sqlalchemy.exc import DBAPIError, OperationalError
 
 from ..celery_app import celery
 from ..models import ImportBatch, ImportSettings, ImportTask
-from ..services.importer import Importer
+from ..services.importer import Importer, ImportResult
+from ..services.library_scan import folder_owner
 from ..services.thumbnailer import Thumbnailer
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -45,8 +46,13 @@ def _mark_failed(session, task, error_msg: str) -> None:
     """Best-effort flip of a 'processing' row to 'failed' + batch counter
     increment. Wrapped in its own try because if the DB is what just
     broke, this commit will also fail — that's why the maintenance sweep
-    exists as a backstop."""
+    exists as a backstop.
+
+    The failure being recorded may have left the session's transaction dead
+    (an error mid-flush), so it is rolled back first; otherwise this commit
+    fails on it too and the row stays 'processing'."""
     try:
+        session.rollback()
         task.status = "failed"
         task.error = error_msg
         task.finished_at = datetime.now(UTC)
@@ -165,6 +171,27 @@ def enqueue_import(task_id: int, task_type: str) -> None:
         import_media_file.delay(task_id)
 
 
+def _attach_where_it_lies(session, importer: Importer, path: Path) -> ImportResult:
+    """Import a file that already sits in its artist's folder (a `library`
+    batch, from services.library_scan). It is attached in place with that
+    artist and source. The post record its folder holds (`_post.json`) is filed
+    first, so the post gets its title and date. Afterwards the file is removed
+    if the library didn't keep it: a copy of something it already holds, or a
+    file the import filters reject. A file whose import crashed stays, with the
+    task marked failed: it may be the only copy of an archive the extractor
+    can't read yet."""
+    if not path.is_file():
+        return ImportResult(status="skipped", error="no longer on disk")
+    artist, source = folder_owner(session, path, IMAGES_ROOT)
+    record = path.parent / "_post.json"
+    if artist is not None and record.is_file():
+        importer.upsert_post_record(record, artist=artist, source=source)
+    result = importer.attach_in_place(path, artist=artist, source=source)
+    if result.status != "failed":
+        importer.discard_after_attach(path, result)
+    return result
+
+
 def _do_import(session, task, import_task_id: int) -> dict:
     """Actual work, called from inside the resilience wrapper."""
     settings = ImportSettings.load_sync(session)
@@ -180,7 +207,10 @@ def _do_import(session, task, import_task_id: int) -> dict:
         deep=deep,
     )
 
-    result = importer.import_one(Path(task.source_path))
+    if batch is not None and batch.scan_mode == "library":
+        result = _attach_where_it_lies(session, importer, Path(task.source_path))
+    else:
+        result = importer.import_one(Path(task.source_path))
 
     if result.status in ("imported", "superseded"):
         task.status = "complete"

backend/app/tasks/maintenance.py

@@ -20,10 +20,12 @@ from ..models import (
     ImportSettings,
     ImportTask,
     LibraryAuditRun,
+    PostAttachment,
     Source,
     TaskRun,
 )
 from ..utils.phash import compute_phash
+from ..utils.staging import is_temp_staging_file
 from ._sync_engine import get_sync_engine
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -81,13 +83,10 @@ WIP_BACKFILL_PAGE = 500
 FFPROBE_TIMEOUT_SECONDS = 10
 TASK_RUN_KEEP_OK_SECONDS = 24 * 3600       # 24 h
 TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600  # 7 days
-# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
-# then os.replace() into place (importer / external_fetch / native_ingest_common /
-# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
-# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
-# keeps it from deleting an in-flight download's staging file mid-write.
+# Orphaned staging files (utils.staging): cleanup_orphaned_temp_files sweeps the
+# ones a killed write left behind; the min-age guard keeps it from deleting an
+# in-flight download's staging file mid-write.
 IMAGES_ROOT = Path("/images")
-TEMP_STAGING_SUFFIXES = (".part", ".partial")
 ORPHAN_TEMP_MIN_AGE_HOURS = 6
 
 # Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
@@ -174,6 +173,9 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # legitimately-running backup isn't flagged.
     "backend.app.tasks.backup.backup_images_task": 420,
     "backend.app.tasks.backup.restore_images_task": 420,
+    # One media-backup chunk stops starting files after 45 min but may finish
+    # a large file in flight, up to its 6.5 h hard limit.
+    "backend.app.tasks.backup.backup_media_task": 420,
     # Library audit scans the full library — 2h hard limit.
     "backend.app.tasks.library_audit.scan_library_for_rule": 130,
     # External file-host fetches (mega/gdrive/film packs) can run to the task's
@@ -191,6 +193,9 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # external-fetch entry above — without an override a healthy in-flight walk
     # is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
     "backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
+    # Importing what's on disk stats every file in the artists' folders, and
+    # its hard limit is 40 min for the same wedged-filesystem case. 45 = 40 + 5.
+    "backend.app.tasks.admin.import_unimported_files_task": 45,
     # Head training and the manual head apply run to 65 min (3900s) — past the
     # ml queue's threshold (#4432). 70 = 65 + 5.
     "backend.app.tasks.ml.train_heads": 70,
@@ -198,6 +203,23 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
 }
 
 
+def _import_queue_depth() -> int | None:
+    """Messages waiting in the broker's import queue, or None when the broker
+    can't be read."""
+    try:
+        import redis
+
+        from ..config import get_config
+
+        client = redis.Redis.from_url(
+            get_config().celery_broker_url, socket_timeout=5, socket_connect_timeout=5,
+        )
+        return int(client.llen("import"))
+    except Exception:  # noqa: BLE001 — an unreadable broker means "don't know"
+        log.warning("recover_interrupted_tasks: import queue depth unreadable", exc_info=True)
+        return None
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
 def recover_interrupted_tasks() -> int:
     """Recover stuck ImportTask rows. Two distinct stuck states:
@@ -213,16 +235,22 @@ def recover_interrupted_tasks() -> int:
        2026-05-24 after a 2224-row zombie pile; archive split out
        2026-05-28.)
 
-    2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
-       creates rows with status='pending' (commit), then in a second pass
-       transitions to 'queued' and calls .delay() (commit). If the scanner
-       crashes between those two commits, rows are orphaned in 'pending'
-       (never enqueued) with no recovery path — invisible to the
-       'processing' sweep above. Flagged 2026-05-25 by operator hitting a
-       5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
-       the operator drains them via /api/import/retry-failed at their own
-       pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
-       import worker.
+    2. 'pending' > 30 min — enqueue-phase crash. scan_directory creates rows
+       with status='pending' (commit), then in a second pass transitions to
+       'queued' and calls .delay() (commit). If the scanner crashes between
+       those two commits, rows are orphaned in 'pending' (never enqueued)
+       with no recovery path — invisible to the 'processing' sweep above.
+       Flip these to 'failed' (not re-enqueue) so the operator drains them
+       via /api/import/retry-failed at their own pace; bulk-re-enqueueing
+       5000+ rows would thundering-herd the import worker.
+
+       A 'queued' row has a broker message, and in a batch larger than half
+       an hour of work it is simply waiting its turn, so its age says
+       nothing. It counts as orphaned only when the broker's import queue
+       is empty: then its message is gone (a broker flush or restart) and
+       nothing will ever run it. When the queue can't be read, queued rows
+       are left alone — a queued row costs nothing to keep, and failing a
+       live one loses its import.
 
     Returns total rows touched (recovered + marked failed).
     """
@@ -299,9 +327,12 @@ def recover_interrupted_tasks() -> int:
         )
         stuck = stuck_result.all()
 
+        orphan_statuses = ["pending"]
+        if _import_queue_depth() == 0:
+            orphan_statuses.append("queued")
         orphan_result = session.execute(
             update(ImportTask)
-            .where(ImportTask.status.in_(["pending", "queued"]))
+            .where(ImportTask.status.in_(orphan_statuses))
             .where(ImportTask.created_at < orphan_cutoff)
             .values(
                 status="failed",
@@ -371,22 +402,58 @@ def cleanup_old_tasks() -> int:
         return result.rowcount or 0
 
 
+def _paths_with_rows(paths: list[str]) -> set[str]:
+    """The subset of `paths` that an image (its file or thumbnail) or an
+    attachment row points at."""
+    owned: set[str] = set()
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        for i in range(0, len(paths), 500):
+            chunk = paths[i:i + 500]
+            owned.update(session.execute(
+                select(ImageRecord.path).where(ImageRecord.path.in_(chunk))
+            ).scalars())
+            owned.update(session.execute(
+                select(ImageRecord.thumbnail_path).where(ImageRecord.thumbnail_path.in_(chunk))
+            ).scalars())
+            owned.update(session.execute(
+                select(PostAttachment.path).where(PostAttachment.path.in_(chunk))
+            ).scalars())
+    return owned
+
+
 @celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
 def cleanup_orphaned_temp_files() -> int:
-    """Delete orphaned .part/.partial staging files under the images root, left by
-    a download/import killed mid-write. Only removes files older than
-    ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
-    pulled out from under it. Returns the count removed."""
+    """Delete orphaned staging files (.part, .partial, megatools' .megatmp.*)
+    under the images root, left by a download/import killed mid-write. Only
+    removes files older than ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight
+    download's staging file is never pulled out from under it, and never a
+    file an image or attachment row points at, whatever its name: that file
+    is library content. Returns the count removed."""
     if not IMAGES_ROOT.is_dir():
         return 0
     cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
-    removed = 0
+    stale: list[Path] = []
     for path in IMAGES_ROOT.rglob("*"):
-        if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
+        if not is_temp_staging_file(path) or not path.is_file():
+            continue
+        try:
+            if path.stat().st_mtime < cutoff:   # fresher may be an active download
+                stale.append(path)
+        except OSError as exc:
+            log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
+    if not stale:
+        return 0
+    owned = _paths_with_rows([str(p) for p in stale])
+    removed = 0
+    for path in stale:
+        if str(path) in owned:
+            log.warning(
+                "cleanup_orphaned_temp_files: keeping %s: a library row points at it",
+                path,
+            )
             continue
         try:
-            if path.stat().st_mtime >= cutoff:
-                continue          # too fresh — may be an active download
             path.unlink()
             removed += 1
         except OSError as exc:
@@ -778,10 +845,18 @@ def recover_stalled_backup_runs() -> int:
             update(BackupRun)
             .where(BackupRun.status.in_(["running", "restoring"]))
             # db backups/restores are fast (40-min window); images run hours (7h).
+            # A media run spans many chunks and hours on purpose, so its window
+            # runs from its last progress, not from its start.
             .where(
                 or_(
                     and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff),
-                    and_(BackupRun.kind != "db", BackupRun.started_at < slow_cutoff),
+                    and_(
+                        BackupRun.kind == "media",
+                        func.coalesce(BackupRun.last_progress_at, BackupRun.started_at)
+                        < slow_cutoff,
+                    ),
+                    and_(BackupRun.kind.not_in(["db", "media"]),
+                         BackupRun.started_at < slow_cutoff),
                 )
             )
             .values(status="error", finished_at=now, error=msg)

backend/app/utils/staging.py

@@ -0,0 +1,20 @@
+"""Downloader staging files.
+
+Downloads and imports write into a staging name and `os.replace()` it into
+place when the write completes (importer, external_fetch, native_ingest_common,
+attachment_store), so a write killed midway leaves a staging file, never a
+truncated final file. Any other file is complete.
+"""
+from pathlib import Path
+
+TEMP_STAGING_SUFFIXES = (".part", ".partial")
+# megatools stages a Mega download as `.megatmp.<id>` in the target dir.
+TEMP_STAGING_PREFIXES = (".megatmp.",)
+
+
+def is_temp_staging_file(path: Path) -> bool:
+    """True for a downloader's staging file: one that never becomes library
+    content under that name, so a stale one is safe to delete."""
+    return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith(
+        TEMP_STAGING_PREFIXES
+    )

extension/background/background.js

@@ -276,11 +276,26 @@ browser.runtime.onMessage.addListener(async (msg) => {
           artistId: msg.artistId ?? null,
           artistName: msg.artistName ?? null,
           usePlatformName: msg.usePlatformName === true,
+          discordAuthors: Array.isArray(msg.discordAuthors) ? msg.discordAuthors : null,
         });
       } catch (e) {
         return { error: e.message };
       }
 
+    case 'DISCORD_POSTERS':
+      try {
+        return await api.getDiscordPosters(msg.url);
+      } catch (e) {
+        return { error: e.message };
+      }
+
+    case 'SET_DISCORD_POSTERS':
+      try {
+        return await api.setDiscordPosters(msg.url, msg.authors || []);
+      } catch (e) {
+        return { error: e.message };
+      }
+
     case 'SEARCH_ARTISTS':
       try {
         return { artists: await api.searchArtists(msg.q || '') };

extension/content/content-script.css

@@ -100,3 +100,7 @@
 /* The panel's own [hidden] — its rows are display:flex, which beats the UA's. */
 .fc-panel [hidden] { display: none !important; }
 .fc-panel__rename { margin-top: 8px; font-size: 13px; }
+/* Poster picker (#4491): who posts in the channel, the likely creator ticked. */
+.fc-panel__posters { max-height: 220px; overflow-y: auto; }
+.fc-panel__poster { flex-wrap: wrap; }
+.fc-panel__poster-meta { flex-basis: 100%; padding-left: 21px; font-size: 11px; color: rgb(170, 166, 156); }

extension/content/content-script.js

@@ -81,6 +81,13 @@
     const probe = currentProbe;
 
     if (probe?.state === 'source_match') {
+      // A followed Discord channel opens its poster list (#4491); the artist
+      // page is one click further, in that panel.
+      if (probe.platform === 'discord' && probe.discord?.channel_id) {
+        if (document.getElementById('fc-add-panel')) closePanel();
+        else openPostersPanel(btn, probe);
+        return;
+      }
       await openArtist(btn, probe.artist?.slug);
       return;
     }
@@ -158,6 +165,120 @@
     document.getElementById('fc-add-panel')?.remove();
   }
 
+  // ---- Poster picker (#4491) ----
+  // Who posted in this channel lately, read by FabledCurator with its Discord
+  // token, the creator the server is named for (or owned by) ticked. The
+  // source keeps their ids: a username can change on a whim, an id can't, so
+  // the list never quietly stops matching. Names are other people's text —
+  // createElement only.
+
+  function posterSection() {
+    const list = el('div', { class: 'fc-panel__posters' }, [
+      el('div', { class: 'fc-panel__hint', text: 'Reading who posts here…' }),
+    ]);
+    const node = el('div', {}, [
+      el('div', { class: 'fc-panel__label', text: 'Only posts by' }),
+      list,
+    ]);
+    const state = { data: null, picked: new Set() };
+
+    function render() {
+      const posters = withSelected(state.data);
+      if (!posters.length) {
+        list.replaceChildren(el('div', { class: 'fc-panel__hint', text: 'No one has posted here lately.' }));
+        return;
+      }
+      const rows = posters.map((p) => {
+        const id = String(p.id);
+        const box = el('input', { type: 'checkbox', checked: state.picked.has(id) });
+        box.addEventListener('change', () => {
+          if (box.checked) state.picked.add(id);
+          else state.picked.delete(id);
+          hint.textContent = pickHint();
+        });
+        const bits = [`${p.images} image${p.images === 1 ? '' : 's'}`, `${p.messages} message${p.messages === 1 ? '' : 's'}`];
+        if (p.reasons?.length) bits.unshift(p.reasons.join(', '));
+        return el('label', { class: 'fc-panel__radio fc-panel__poster' }, [
+          box,
+          el('span', { text: posterLabel(p) }),
+          el('span', { class: 'fc-panel__poster-meta', text: bits.join(' · ') }),
+        ]);
+      });
+      const hint = el('div', { class: 'fc-panel__hint', text: pickHint() });
+      list.replaceChildren(...rows, hint);
+    }
+
+    function pickHint() {
+      return state.picked.size
+        ? 'Only the ticked people’s posts are taken.'
+        : 'Nothing ticked: everyone’s posts are taken.';
+    }
+
+    async function load() {
+      let r;
+      try {
+        r = await browser.runtime.sendMessage({ type: 'DISCORD_POSTERS', url: window.location.href });
+      } catch (e) {
+        r = { error: e.message };
+      }
+      if (r?.error) {
+        list.replaceChildren(el('div', { class: 'fc-panel__hint', text: `Couldn’t list posters — ${r.error}` }));
+        return;
+      }
+      state.data = r;
+      state.picked = initialPosterPick(r);
+      render();
+    }
+
+    // null until the list has loaded: an add made before then leaves the
+    // source's list as it is instead of clearing it.
+    const request = () => (state.data ? postersRequest(state.data.posters, state.picked) : null);
+    return { node, load, request };
+  }
+
+  function openPostersPanel(btn, probe) {
+    closePanel();
+    const d = probe.discord || {};
+    const posters = posterSection();
+    const saveBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Save' });
+    const artistBtn = el('button', { class: 'fc-panel__btn', text: `Open ${probe.artist?.name || 'artist'}` });
+    const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Close' });
+    const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
+      el('div', { class: 'fc-panel__title', text: 'Discord source' }),
+      el('div', { class: 'fc-panel__sub', text: d.channel_id ? channelLabel(d) : serverLabel(d) }),
+      posters.node,
+      el('div', { class: 'fc-panel__actions' }, [artistBtn, cancelBtn, saveBtn]),
+    ]);
+    panel.addEventListener('keydown', (e) => {
+      e.stopPropagation();
+      if (e.key === 'Escape') closePanel();
+    });
+    cancelBtn.addEventListener('click', closePanel);
+    artistBtn.addEventListener('click', () => { closePanel(); openArtist(btn, probe.artist?.slug); });
+    saveBtn.addEventListener('click', async () => {
+      const authors = posters.request();
+      if (!authors) return;
+      saveBtn.disabled = true;
+      try {
+        const r = await browser.runtime.sendMessage({
+          type: 'SET_DISCORD_POSTERS', url: window.location.href, authors,
+        });
+        if (r?.error) {
+          showToast(`Error: ${r.error}`, 'error');
+          return;
+        }
+        showToast(authors.length ? `Only posts by ${authors.map((a) => a.name || a.id).join(', ')}` : 'Taking everyone’s posts', 'success');
+        closePanel();
+      } catch (e) {
+        showToast(`Error: ${e.message}`, 'error');
+      } finally {
+        saveBtn.disabled = false;
+      }
+    });
+    document.body.appendChild(panel);
+    posters.load();
+  }
+
   async function openAddPanel(btn, probe) {
     closePanel();
     const platformName = PLATFORMS[probe.platform]?.name || probe.platform;
@@ -184,6 +305,8 @@
     const d = probe.discord || {};
     const discord = probe.platform === 'discord';
     const choice = panelDefaults(probe, window.location.href);
+    // #4491: on a channel page, pick whose posts the new source takes.
+    const posters = discord && d.channel_id ? posterSection() : null;
 
     const scopeRow = (value, label, disabled) => {
       const input = el('input', {
@@ -234,6 +357,7 @@
       el('div', { class: 'fc-panel__label', text: 'Artist' }),
       el('div', { class: 'fc-panel__combo' }, [nameInput, results]),
       renameRow,
+      ...(posters ? [posters.node] : []),
       hint,
       el('div', { class: 'fc-panel__actions' }, [cancelBtn, addBtn]),
     ]);
@@ -392,6 +516,8 @@
     addBtn.addEventListener('click', async () => {
       const req = addRequest(choice);
       if (!req) return;
+      const authors = posters?.request();
+      if (authors) req.discordAuthors = authors;
       const btn = document.getElementById('fc-add-source-btn');
       addBtn.disabled = true;
       const ok = await add(btn || addBtn, req);
@@ -402,6 +528,7 @@
     document.body.appendChild(panel);
     refresh();
     nameInput.focus();
+    posters?.load();
     nameInput.select();
     // Search what the field opens with — the server's name, usually — so an
     // artist it already matches is picked before the operator types anything.

extension/lib/api.js

@@ -106,8 +106,11 @@ class FabledCuratorAPI {
   // artist from the URL. A Discord channel always sends one.
   // usePlatformName: a Patreon source joining an existing artist renames it
   // to the Patreon display name (Patreon is canon; name only, never the slug).
-  quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false } = {}) {
+  // discordAuthors ([{id, name}], #4491): a Discord source's poster list, set
+  // in the same step.
+  quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false, discordAuthors = null } = {}) {
     const body = { url };
+    if (discordAuthors) body.discord_authors = discordAuthors;
     if (artistId != null) body.artist_id = artistId;
     else if (artistName) body.artist_name = artistName;
     if (usePlatformName) body.use_platform_name = true;
@@ -123,6 +126,15 @@ class FabledCuratorAPI {
     const qs = new URLSearchParams(params).toString();
     return this.request('GET', `/extension/probe?${qs}`);
   }
+  // #4491: who posts in the Discord channel `url` shows, lately, with the
+  // likely creator marked; and setting the list on the source that follows it.
+  getDiscordPosters(url) {
+    const qs = new URLSearchParams({ url }).toString();
+    return this.request('GET', `/extension/discord/posters?${qs}`);
+  }
+  setDiscordPosters(url, authors) {
+    return this.request('POST', '/extension/discord/posters', { url, authors });
+  }
   // Latest published extension version on this instance — drives the in-app
   // update prompt. Public endpoint (no key needed, but request() sends it
   // harmlessly). Returns {version, xpi_url, latest_url, sha256}.

extension/lib/chip.js

@@ -144,3 +144,49 @@ function inlineCompletion(typed, results) {
   if (!t) return null;
   return (results || []).find((a) => a.name.length > t.length && a.name.toLowerCase().startsWith(t)) || null;
 }
+
+// ---- Discord poster picker (#4491) ----
+// /api/extension/discord/posters lists who posted in the channel lately, with
+// the creator the server is named for (or owned by) marked `suggested`. The
+// picker ticks posters; the source keeps their ids, since a username can
+// change on a whim and an id can't.
+
+/** "Todding (@todding)" — the display name, then the handle when it differs. */
+function posterLabel(p) {
+  const shown = p?.global_name || p?.username || p?.id || '';
+  const handle = p?.username && p.username !== shown ? ` (@${p.username})` : '';
+  return `${shown}${handle}`;
+}
+
+/** What the picker starts with: the source's current list when it has one,
+ *  else the suggested creator(s). A set of ids. */
+function initialPosterPick(data) {
+  const current = (data?.selected || []).map(String);
+  if (current.length) return new Set(current);
+  return new Set((data?.posters || []).filter((p) => p.suggested).map((p) => String(p.id)));
+}
+
+/** The checklist's rows: who posted lately, then anyone already on the
+ *  source's list who hasn't — by the name they were picked under — so they
+ *  can still be unticked. */
+function withSelected(data) {
+  const posters = [...(data?.posters || [])];
+  const shown = new Set(posters.map((p) => String(p.id)));
+  const labels = data?.selected_labels || {};
+  for (const id of (data?.selected || []).map(String)) {
+    if (shown.has(id)) continue;
+    posters.push({ id, global_name: labels[id] || null, username: null,
+      messages: 0, images: 0, reasons: ['on the list, not posting lately'] });
+  }
+  return posters;
+}
+
+/** The picked posters as the API takes them: [{id, name}], in list order.
+ *  An id already on the source but no longer posting lately is kept. */
+function postersRequest(posters, picked) {
+  const byId = new Map((posters || []).map((p) => [String(p.id), p]));
+  return [...picked].map((id) => ({
+    id,
+    name: byId.has(id) ? (byId.get(id).global_name || byId.get(id).username || id) : null,
+  }));
+}

extension/test/chip.spec.js

@@ -198,3 +198,46 @@ describe('Add panel on Patreon and SubscribeStar', () => {
     expect(c.artistName).toBe('Tamada')
   })
 })
+
+const { posterLabel, initialPosterPick, postersRequest, withSelected } = loadLib('chip.js', [
+  'posterLabel', 'initialPosterPick', 'postersRequest', 'withSelected',
+])
+
+describe('the poster checklist rows', () => {
+  it('adds someone on the list who has not posted lately, by their saved name', () => {
+    const rows = withSelected({
+      posters: [{ id: '7', username: 'todding', global_name: 'Todding' }],
+      selected: ['7', '42'],
+      selected_labels: { 42: 'Old Name' },
+    })
+    expect(rows.map((r) => r.id)).toEqual(['7', '42'])
+    expect(posterLabel(rows[1])).toBe('Old Name')
+    expect(rows[1].reasons).toEqual(['on the list, not posting lately'])
+  })
+})
+
+describe('the Discord poster picker (#4491)', () => {
+  const posters = [
+    { id: '7', username: 'todding', global_name: 'Todding', suggested: true },
+    { id: '8', username: 'jakeboii', global_name: 'Jake Boii', suggested: false },
+    { id: '9', username: 'plain', global_name: null, suggested: false },
+  ]
+
+  it('labels a poster by display name, then handle', () => {
+    expect(posterLabel(posters[0])).toBe('Todding (@todding)')
+    expect(posterLabel(posters[2])).toBe('plain')
+  })
+
+  it('starts from the source list when it has one, else the suggestion', () => {
+    expect([...initialPosterPick({ posters, selected: [] })]).toEqual(['7'])
+    expect([...initialPosterPick({ posters, selected: ['8'] })]).toEqual(['8'])
+    expect([...initialPosterPick({ posters: [] })]).toEqual([])
+  })
+
+  it('sends ids with the names they were picked under, keeping unknown ids', () => {
+    expect(postersRequest(posters, new Set(['7', '42']))).toEqual([
+      { id: '7', name: 'Todding' },
+      { id: '42', name: null },
+    ])
+  })
+})

frontend/src/components/cleanup/SingleColorAuditCard.vue

@@ -101,7 +101,7 @@ import MaintenanceTile from '../common/MaintenanceTile.vue'
 import { useCleanupStore } from '../../stores/cleanup.js'
 
 const store = useCleanupStore()
-const threshold = ref(0.95)
+const threshold = ref(0.995)
 const tolerance = ref(30)
 const audit = ref(null)
 const busy = ref(false)

frontend/src/components/posts/PostCard.vue

@@ -26,6 +26,7 @@
         :to="{ name: 'artist', params: { slug: post.artist.slug } }"
         class="fc-post-card__artist"
       >{{ post.artist.name }}</RouterLink>
+      <span v-if="post.channel" class="fc-post-card__meta">#{{ post.channel }}</span>
       <span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
       <span v-if="totalImages" class="fc-post-card__meta">
         · {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}

frontend/src/components/settings/BackupCard.vue

@@ -4,11 +4,18 @@
     title="Backups"
     blurb="Database + image backups, schedules and restore."
   >
-      <p class="fc-muted text-body-2 mb-4">
+      <p v-if="offsite" class="fc-muted text-body-2 mb-4">
+        Database dumps stream to the <strong>{{ offsite.bucket }}</strong> bucket
+        at {{ offsite.endpoint }}, with no copy kept on this server; the bucket's
+        own retention thins old dumps. Images and attachments go to the same
+        bucket's <code>media/</code> folder, below.
+      </p>
+      <p v-else class="fc-muted text-body-2 mb-4">
         pg_dump for the database (fast, nightly-schedulable);
         tar+zstd for the images (slow, manual only). Files live in
         <code>/images/_backups/</code>. Tag a backup to protect it
-        from autoprune.
+        from autoprune. Set up an off-site destination above to send
+        database dumps off this server instead.
       </p>
 
       <v-alert
@@ -40,6 +47,10 @@
           @update:model-value="onSettingChange('backup_db_keep_last_n', Number($event))"
         />
       </div>
+      <p v-if="offsite" class="fc-muted text-caption mb-3">
+        Nightly sends the database, then any new images and attachments. Keep
+        last applies to this list and to any dumps still on this server.
+      </p>
       <v-btn
         color="accent" variant="flat" rounded="pill"
         prepend-icon="mdi-database-arrow-up" class="mb-3"
@@ -56,6 +67,38 @@
 
       <v-divider class="my-4" />
 
+      <!-- Off-site: images and attachments as one file per fingerprint -->
+      <template v-if="offsite">
+        <h3 class="fc-section-title">Images and attachments</h3>
+        <p class="fc-muted text-body-2 mb-2">
+          Each file goes to the bucket's <code>media/</code> folder once, named by
+          its fingerprint, and the bucket checks it against the fingerprint taken
+          at import. Only files not sent yet go in each run. With Nightly on, they
+          follow the database dump every night.
+        </p>
+        <div v-if="coverage" class="mb-3">
+          <div class="text-body-2 mb-1" style="font-variant-numeric: tabular-nums;">
+            {{ coverage.files_sent.toLocaleString() }} of
+            {{ coverage.files_total.toLocaleString() }} files in the bucket
+            ({{ humanBytes(coverage.bytes_sent) }} of {{ humanBytes(coverage.bytes_total) }})
+          </div>
+          <v-progress-linear
+            :model-value="coveragePct" color="accent" height="6" rounded
+          />
+        </div>
+        <v-btn
+          color="accent" variant="flat" rounded="pill"
+          prepend-icon="mdi-cloud-upload-outline" class="mb-3"
+          :loading="mediaTriggering" :disabled="mediaRunning"
+          @click="onTrigger('media')"
+        >Send new files now</v-btn>
+        <span v-if="mediaRunning" class="text-caption fc-muted ml-2">
+          A run is going: {{ runningSent.toLocaleString() }} files sent so far.
+        </span>
+        <BackupRunsTable :runs="store.mediaRuns" />
+      </template>
+
+      <template v-else>
       <!-- Images section -->
       <h3 class="fc-section-title">Images</h3>
       <div class="fc-settings-row mb-3">
@@ -82,6 +125,7 @@
         @delete="onDelete"
         @tag="onTag"
       />
+      </template>
     <BackupConfirmModal
       v-model="confirmOpen"
       :action="confirmAction"
@@ -99,6 +143,7 @@ import { computed, onMounted, onUnmounted, ref } from 'vue'
 import { useBackupStore } from '../../stores/backup.js'
 import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
 import MaintenanceTile from '../common/MaintenanceTile.vue'
+import { humanBytes } from '../../utils/bytes.js'
 import BackupRunsTable from './BackupRunsTable.vue'
 
 const store = useBackupStore()
@@ -110,6 +155,21 @@ const settings = computed(() => store.settings || {
   backup_images_keep_last_n: 3,
 })
 
+// The off-site destination, once it is fully set up: database dumps go there.
+const offsite = computed(() => (
+  store.destination?.configured ? store.destination : null
+))
+
+const coverage = computed(() => store.mediaCoverage)
+const coveragePct = computed(() => {
+  const c = coverage.value
+  return c && c.files_total ? (100 * c.files_sent) / c.files_total : 0
+})
+const runningMedia = computed(() => store.mediaRuns.find((r) => r.status === 'running'))
+const mediaRunning = computed(() => Boolean(runningMedia.value))
+const runningSent = computed(() => runningMedia.value?.manifest?.sent || 0)
+
+const mediaTriggering = ref(false)
 const dbTriggering = ref(false)
 const imagesTriggering = ref(false)
 const confirmOpen = ref(false)
@@ -119,14 +179,24 @@ const confirmRunId = ref(0)
 const confirmDescription = ref('')
 
 let pollId = null
+let polls = 0
+// Coverage counts every library file, so it refreshes every sixth poll
+// (30 s) rather than with the run lists.
+const COVERAGE_EVERY = 6
 function pollOnce() {
   if (document.hidden) return
   store.loadRuns('db')
-  store.loadRuns('images')
+  if (offsite.value) {
+    store.loadRuns('media')
+    if (polls++ % COVERAGE_EVERY === 0) store.loadMediaCoverage()
+  } else {
+    store.loadRuns('images')
+  }
 }
 
 onMounted(() => {
   store.loadSettings()
+  if (!store.destination) store.loadDestination().catch(() => {})
   pollOnce()
   pollId = setInterval(pollOnce, 5000)
 })
@@ -135,7 +205,7 @@ onUnmounted(() => {
 })
 
 async function onTrigger(kind) {
-  const triggeringRef = kind === 'db' ? dbTriggering : imagesTriggering
+  const triggeringRef = { db: dbTriggering, images: imagesTriggering, media: mediaTriggering }[kind]
   triggeringRef.value = true
   try {
     await store.triggerBackup(kind)

frontend/src/components/settings/BackupDestinationCard.vue

@@ -0,0 +1,217 @@
+<!--
+  Off-site backup destination: an S3 bucket (Fabled Archives or any S3
+  service) that backups are sent to. The five fields only work together, so
+  they save as one with an explicit button rather than field by field. The
+  secret key is write-only: once saved it is never shown again, and leaving the
+  field blank keeps it. Test destination lists the bucket with what is typed
+  in the form (saved or not) and writes nothing.
+-->
+<template>
+  <MaintenanceTile
+    icon="mdi-cloud-upload-outline"
+    title="Off-site backup destination"
+    blurb="The S3 bucket (Fabled Archives or any S3 service) that backups are sent to."
+  >
+    <p class="fc-muted text-body-2 mb-3">
+      Backups sent here survive losing this server. FabledCurator only adds to the
+      bucket and never deletes from it, so a key that may add and read objects but
+      not delete them is enough, and is the safer key to use. Account logins are
+      never sent.
+    </p>
+
+    <v-alert
+      v-if="loadError" type="warning" variant="tonal" density="compact" class="mb-3"
+    >{{ loadError }}</v-alert>
+
+    <v-text-field
+      v-model="form.endpoint" label="Address"
+      placeholder="http://archives.example:8081" density="compact"
+      hint="Include the port the S3 service listens on." persistent-hint
+      class="mb-3" :disabled="busy"
+    />
+    <div class="fc-dest-row mb-3">
+      <v-text-field
+        v-model="form.bucket" label="Bucket" density="compact" hide-details
+        :disabled="busy"
+      />
+      <v-text-field
+        v-model="form.region" label="Region" density="compact" hide-details
+        class="fc-dest-region" :disabled="busy"
+      />
+    </div>
+    <v-text-field
+      v-model="form.access_key" label="Access key" density="compact" hide-details
+      autocomplete="off" class="mb-3" :disabled="busy"
+    />
+    <v-text-field
+      v-model="form.secret_key" label="Secret key" type="password"
+      autocomplete="new-password" density="compact"
+      :placeholder="secretSet ? 'Saved. Leave blank to keep it.' : ''"
+      :persistent-placeholder="secretSet"
+      :hint="secretSet ? 'A saved secret is never shown again.' : ''" persistent-hint
+      class="mb-3" :disabled="busy"
+    />
+    <v-text-field
+      v-model.number="form.upload_limit_mbps" label="Upload limit (Mbit/s)"
+      type="number" min="0" density="compact" class="fc-dest-limit mb-3"
+      hint="0 for no limit. A lower limit leaves more of the link free, and makes each upload take longer."
+      persistent-hint :disabled="busy"
+    />
+
+    <div class="d-flex align-center flex-wrap mb-2" style="gap: 8px;">
+      <v-btn
+        size="small" variant="flat" color="accent" rounded="pill"
+        prepend-icon="mdi-content-save" :loading="saving"
+        :disabled="busy || !dirty" @click="onSave"
+      >Save</v-btn>
+      <v-btn
+        size="small" variant="tonal" rounded="pill"
+        prepend-icon="mdi-lan-connect" :loading="testing"
+        :disabled="busy || !form.endpoint" @click="onTest"
+      >Test destination</v-btn>
+      <v-btn
+        v-if="configured" size="small" variant="text" rounded="pill" color="error"
+        :disabled="busy" @click="confirmRemove = true"
+      >Remove</v-btn>
+      <span v-if="savedNote" class="text-caption text-success">{{ savedNote }}</span>
+    </div>
+
+    <v-alert
+      v-if="testResult" :type="testResult.ok ? 'success' : 'error'"
+      variant="tonal" density="compact" class="mb-1"
+    >{{ testResult.message }}</v-alert>
+    <p v-if="!configured && !testResult" class="fc-muted text-caption mb-0">
+      No destination is set up, so nothing is sent off this server yet.
+    </p>
+    <p v-else-if="configured && !testResult" class="fc-muted text-caption mb-0">
+      Database backups and the library's images and attachments go to this bucket.
+      Set when they run under Backups below.
+    </p>
+
+    <v-dialog v-model="confirmRemove" max-width="440">
+      <v-card>
+        <v-card-title class="text-h6">Remove the destination?</v-card-title>
+        <v-card-text class="text-body-2">
+          FabledCurator forgets the address and both keys and stops sending
+          backups there. What was already sent stays in the bucket.
+        </v-card-text>
+        <v-card-actions>
+          <v-spacer />
+          <v-btn variant="text" @click="confirmRemove = false">Cancel</v-btn>
+          <v-btn color="error" variant="flat" :loading="saving" @click="onRemove">
+            Remove
+          </v-btn>
+        </v-card-actions>
+      </v-card>
+    </v-dialog>
+  </MaintenanceTile>
+</template>
+
+<script setup>
+import { computed, onMounted, reactive, ref } from 'vue'
+
+import { useBackupStore } from '../../stores/backup.js'
+import MaintenanceTile from '../common/MaintenanceTile.vue'
+
+const store = useBackupStore()
+
+const FIELDS = ['endpoint', 'bucket', 'access_key', 'region']
+const DEFAULT_LIMIT = 200
+const form = reactive({
+  endpoint: '', bucket: '', access_key: '', region: 'us-east-1', secret_key: '',
+  upload_limit_mbps: DEFAULT_LIMIT,
+})
+const saved = ref(null)
+const loadError = ref('')
+const saving = ref(false)
+const testing = ref(false)
+const testResult = ref(null)
+const savedNote = ref('')
+const confirmRemove = ref(false)
+
+const busy = computed(() => saving.value || testing.value)
+const secretSet = computed(() => Boolean(saved.value?.secret_set))
+const configured = computed(() => Boolean(saved.value?.configured))
+const dirty = computed(() => (
+  !saved.value
+  || form.secret_key.trim() !== ''
+  || FIELDS.some((f) => (form[f] || '').trim() !== (saved.value[f] || ''))
+  || form.upload_limit_mbps !== saved.value.upload_limit_mbps
+))
+
+function apply(dest) {
+  saved.value = dest
+  for (const f of FIELDS) form[f] = dest[f] || (f === 'region' ? 'us-east-1' : '')
+  form.upload_limit_mbps = dest.upload_limit_mbps ?? DEFAULT_LIMIT
+  form.secret_key = ''
+}
+
+function fields() {
+  const out = {}
+  for (const f of FIELDS) out[f] = form[f]
+  if (form.secret_key.trim()) out.secret_key = form.secret_key
+  const limit = Number(form.upload_limit_mbps)
+  if (Number.isInteger(limit) && limit >= 0) out.upload_limit_mbps = limit
+  return out
+}
+
+function messageOf(e) {
+  return e?.body?.detail || e?.message || 'The request failed.'
+}
+
+onMounted(async () => {
+  try {
+    apply(await store.loadDestination())
+  } catch (e) {
+    loadError.value = `Couldn't load the destination: ${messageOf(e)}`
+  }
+})
+
+async function onSave() {
+  saving.value = true
+  savedNote.value = ''
+  testResult.value = null
+  try {
+    apply(await store.saveDestination(fields()))
+    savedNote.value = 'Saved.'
+  } catch (e) {
+    testResult.value = { ok: false, message: `Not saved: ${messageOf(e)}` }
+  } finally {
+    saving.value = false
+  }
+}
+
+async function onTest() {
+  testing.value = true
+  savedNote.value = ''
+  testResult.value = null
+  try {
+    testResult.value = await store.testDestination(fields())
+  } catch (e) {
+    testResult.value = { ok: false, message: messageOf(e) }
+  } finally {
+    testing.value = false
+  }
+}
+
+async function onRemove() {
+  saving.value = true
+  try {
+    apply(await store.removeDestination())
+    testResult.value = null
+    savedNote.value = 'Removed.'
+    confirmRemove.value = false
+  } catch (e) {
+    testResult.value = { ok: false, message: `Not removed: ${messageOf(e)}` }
+  } finally {
+    saving.value = false
+  }
+}
+</script>
+
+<style scoped>
+.fc-dest-row { display: flex; gap: 12px; flex-wrap: wrap; }
+.fc-dest-row > :first-child { flex: 1 1 200px; }
+.fc-dest-region { flex: 0 1 160px; }
+.fc-dest-limit { max-width: 260px; }
+</style>

frontend/src/components/settings/BackupRunsTable.vue

@@ -4,6 +4,7 @@
       <tr>
         <th>When</th>
         <th>Status</th>
+        <th>Saved to</th>
         <th class="text-right">Duration</th>
         <th class="text-right">Size</th>
         <th>Tag</th>
@@ -20,6 +21,15 @@
             {{ statusIcon(r.status) }}
           </v-icon>
           {{ r.status }}
+          <div
+            v-if="r.error" class="fc-run-error text-caption text-error" :title="r.error"
+          >{{ r.error }}</div>
+        </td>
+        <td :title="r.offsite ? `${r.offsite.endpoint} · ${r.offsite.key}` : ''">
+          <template v-if="r.offsite">
+            <v-icon size="small" class="mr-1">mdi-cloud-check-outline</v-icon>{{ r.offsite.bucket }}
+          </template>
+          <span v-else class="fc-muted">This server</span>
         </td>
         <td class="text-right fc-tabular">{{ formatDuration(r) }}</td>
         <td class="text-right fc-tabular">{{ formatBytes(r.size_bytes) }}</td>
@@ -30,7 +40,8 @@
           <span v-else class="fc-muted">—</span>
         </td>
         <td class="text-right">
-          <KebabMenu :label="`Actions for backup ${r.id}`">
+          <span v-if="r.offsite" class="fc-muted">—</span>
+          <KebabMenu v-else :label="`Actions for backup ${r.id}`">
             <v-list-item
               :title="r.tag ? 'Untag' : 'Tag…'"
               @click="$emit('tag', r)"
@@ -47,7 +58,7 @@
         </td>
       </tr>
       <tr v-if="!runs.length">
-        <td colspan="6" class="text-center fc-muted py-4">No backups yet.</td>
+        <td colspan="7" class="text-center fc-muted py-4">No backups yet.</td>
       </tr>
     </tbody>
   </v-table>
@@ -97,4 +108,7 @@ function formatRelative(iso) {
 <style scoped>
 .fc-backup-table { background: transparent; }
 .fc-tabular { font-variant-numeric: tabular-nums; }
+.fc-run-error {
+  max-width: 320px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
+}
 </style>

frontend/src/components/settings/ImportFiltersForm.vue

@@ -77,7 +77,7 @@
         <v-col cols="12" sm="6">
           <v-slider
             v-model="local.single_color_threshold" label="Single-color threshold"
-            min="0.5" max="1" step="0.05" thumb-label hide-details color="accent"
+            min="0.9" max="1" step="0.005" thumb-label hide-details color="accent"
             :disabled="!local.skip_single_color" @end="save"
           />
         </v-col>
@@ -142,7 +142,7 @@ const PHASH_TICKS = { 0: 'Exact', 12: 'Strict', 24: 'Default', 48: 'Loose' }
 const local = reactive({
   min_width: 0, min_height: 0,
   skip_transparent: false, transparency_threshold: 0.9,
-  skip_single_color: false, single_color_threshold: 0.95,
+  skip_single_color: false, single_color_threshold: 0.995,
   phash_threshold: 24,
   wip_title_tagging_enabled: true,
 })

frontend/src/components/settings/LibraryImportCard.vue

@@ -0,0 +1,150 @@
+<template>
+  <!-- Files a download wrote into an artist's folder but never imported: a run
+       that stopped between writing and importing leaves them there, and the
+       walk that marked them seen never returns. Preview lists them per artist;
+       Import queues them as an import batch that attaches each file where it
+       lies. -->
+  <MaintenanceTile
+    icon="mdi-folder-download"
+    title="Import files the library is missing"
+    blurb="Find art in the artists' folders that was downloaded but never imported, and import it where it is."
+    :open="applying || previewing"
+  >
+    <p class="text-body-2 mb-3">
+      A download writes its files into the artist's folder, then imports them.
+      A run that stops in between leaves complete files that no image points
+      at, and later downloads skip them as already seen. This walks every
+      artist's folder for images, videos and archives the library has
+      <strong>no record of</strong>. <strong>Preview</strong> first;
+      <strong>Import</strong> runs them through the normal import where they
+      lie. A file that turns out to be a copy of something already in the
+      library, or that the import filters reject, is removed from disk. Files
+      changed in the last day are left alone, so a download still running is
+      never caught mid-write.
+    </p>
+
+    <div class="d-flex align-center flex-wrap" style="gap: 12px;">
+      <v-btn
+        color="primary" variant="tonal" rounded="pill"
+        :loading="previewing" :disabled="applying" @click="preview"
+      >
+        <v-icon start>mdi-magnify</v-icon> Preview
+      </v-btn>
+      <v-btn
+        color="primary" rounded="pill"
+        :loading="applying"
+        :disabled="previewing || !canApply"
+        @click="confirmOpen = true"
+      >
+        <v-icon start>mdi-folder-download</v-icon> Import
+      </v-btn>
+    </div>
+
+    <v-alert
+      v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
+      density="comfortable"
+    >
+      <span v-if="applied && summary.batch_id">
+        Queued {{ summary.files }} file(s) ({{ humanBytes(summary.bytes) }}) for
+        import. Progress shows in the import queue below and on Activity.
+      </span>
+      <span v-else-if="hasWork">
+        {{ summary.files }} file(s), {{ humanBytes(summary.bytes) }}, are on
+        disk but not in the library<span v-if="summary.archives">, including
+        {{ summary.archives }} archive(s)</span>.
+        Click <strong>Import</strong> to bring them in.
+      </span>
+      <span v-else>Nothing to import: every file in the artists' folders is in the library.</span>
+
+      <div v-if="passedOver" class="mt-1 text-caption">
+        Left alone: {{ summary.recent }} changed in the last day,
+        {{ summary.not_media }} that aren't images, videos or archives, and
+        {{ summary.no_artist }} in folders that match no artist.
+      </div>
+    </v-alert>
+
+    <v-table v-if="hasWork && !applied" density="compact" class="mt-3">
+      <thead>
+        <tr>
+          <th>Artist</th>
+          <th class="text-right">Files</th>
+          <th class="text-right">Size</th>
+          <th>For example</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr v-for="a in summary.artists" :key="a.slug">
+          <td>{{ a.slug }}</td>
+          <td class="text-right">{{ a.files }}</td>
+          <td class="text-right">{{ humanBytes(a.bytes) }}</td>
+          <td class="text-caption text-medium-emphasis fc-sample">{{ a.samples[0] }}</td>
+        </tr>
+      </tbody>
+    </v-table>
+
+    <QueueStatusBar queue="import" queue-label="Import" />
+
+    <v-dialog v-model="confirmOpen" max-width="440">
+      <v-card>
+        <v-card-title>Import {{ summary?.files ?? 0 }} file(s)?</v-card-title>
+        <v-card-text class="text-body-2">
+          This imports <strong>{{ summary?.files ?? 0 }}</strong> file(s)
+          ({{ humanBytes(summary?.bytes) }}) where they lie. Any that turn out
+          to be copies of art already in the library, or that the import
+          filters reject, are removed from disk.
+        </v-card-text>
+        <v-card-actions>
+          <v-spacer />
+          <v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
+          <v-btn color="primary" @click="apply">Import</v-btn>
+        </v-card-actions>
+      </v-card>
+    </v-dialog>
+  </MaintenanceTile>
+</template>
+
+<script setup>
+import { computed, ref } from 'vue'
+
+import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
+import { humanBytes } from '../../utils/bytes.js'
+import MaintenanceTile from '../common/MaintenanceTile.vue'
+import QueueStatusBar from './QueueStatusBar.vue'
+
+const confirmOpen = ref(false)
+
+// The walk stats every file in the artists' folders, so on a large library on
+// network storage it can take many minutes. 900 polls × 2s = 30 min, the
+// task's own soft limit.
+const { previewing, applying, summary, applied, preview, apply: applyTask } = useMaintenanceTask({
+  endpoint: '/api/admin/maintenance/import-unimported',
+  storageKey: 'fc.maint.importUnimported',
+  appliedToast: 'Files queued for import',
+  maxPolls: 900,
+})
+
+const hasWork = computed(() => !!summary.value && summary.value.files > 0)
+const canApply = computed(() => hasWork.value && !applied.value)
+const passedOver = computed(() => {
+  const s = summary.value
+  return !!s && (s.recent > 0 || s.not_media > 0 || s.no_artist > 0)
+})
+const summaryType = computed(() => {
+  if (applied.value) return 'success'
+  return hasWork.value ? 'info' : 'success'
+})
+
+function apply () {
+  confirmOpen.value = false
+  applyTask()
+}
+</script>
+
+<style scoped>
+.fc-sample {
+  max-width: 320px;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+</style>

frontend/src/components/settings/MaintenancePanel.vue

@@ -63,8 +63,11 @@
 
     <section class="fc-section">
       <h3 class="fc-section__title">Storage</h3>
-      <p class="fc-section__hint">Database + image backups and restore.</p>
+      <p class="fc-section__hint">
+        Database + image backups and restore, and where backups are sent off this server.
+      </p>
       <div class="fc-tile-stack">
+        <BackupDestinationCard />
         <BackupCard />
       </div>
     </section>
@@ -93,6 +96,7 @@ import PostAssociationsCard from './PostAssociationsCard.vue'
 import GpuAgentCard from './GpuAgentCard.vue'
 import AliasTable from './AliasTable.vue'
 import BackupCard from './BackupCard.vue'
+import BackupDestinationCard from './BackupDestinationCard.vue'
 import { useSystemActivityStore } from '../../stores/systemActivity.js'
 
 // Poll queue depths so each task tile's QueueStatusBar shows live pending

frontend/src/components/subscriptions/PosterCleanupDialog.vue

@@ -0,0 +1,135 @@
+<template>
+  <!-- #4486. A Discord source's posts by people outside its "Only posts by"
+       list, taken before the list existed. Preview first, always: nothing is
+       deleted until the operator has seen the per-poster breakdown and typed
+       the token that preview returned. -->
+  <v-dialog :model-value="modelValue" max-width="560"
+            @update:model-value="$emit('update:modelValue', $event)">
+    <v-card>
+      <v-card-title>Remove posts from other posters</v-card-title>
+      <v-card-text>
+        <div class="text-caption mb-2">{{ source.display_name || source.url }}</div>
+        <v-progress-linear v-if="loading" indeterminate color="accent" class="mb-3" />
+        <v-alert v-else-if="error" type="warning" variant="tonal" density="compact">
+          {{ error }}
+        </v-alert>
+        <template v-else-if="result">
+          <v-alert type="success" variant="tonal" density="compact">
+            Removed {{ result.posts_deleted }} post(s), {{ result.images_deleted }} image(s),
+            {{ result.attachments_deleted }} attachment(s) and {{ result.drops_deleted }}
+            grouped post(s). Grouping rebuilds what remains on its next sweep.
+          </v-alert>
+        </template>
+        <template v-else-if="preview">
+          <p class="mb-2">
+            Keeping posts by <strong>{{ preview.authors.join(', ') }}</strong>.
+          </p>
+          <p v-if="!preview.posts" class="fc-dim">
+            Nothing to remove: every post with a known poster is by someone on the list.
+          </p>
+          <template v-else>
+            <v-table density="compact" class="mb-3">
+              <thead><tr><th>Poster</th><th class="text-right">Posts</th></tr></thead>
+              <tbody>
+                <tr v-for="p in preview.posters" :key="p.poster">
+                  <td>{{ p.poster }}</td>
+                  <td class="text-right">{{ p.posts }}</td>
+                </tr>
+              </tbody>
+            </v-table>
+            <p class="mb-1">
+              {{ preview.posts }} post(s), {{ preview.images }} image(s) found only on them,
+              {{ preview.attachments }} attachment(s).
+            </p>
+            <p v-if="preview.drops" class="mb-1">
+              {{ preview.drops }} grouped post(s) that include them are ungrouped; the
+              rest regroup on the next sweep.
+            </p>
+            <p class="fc-dim">
+              Images that also belong to a kept post stay. If a name you want to keep
+              appears above, add it to the source's "Only posts by" list first.
+            </p>
+          </template>
+          <p v-if="preview.unknown_posts" class="fc-dim mt-2">
+            {{ preview.unknown_posts }} older post(s) don't record who posted them and
+            are left alone.
+          </p>
+        </template>
+      </v-card-text>
+      <v-card-actions>
+        <v-spacer />
+        <v-btn variant="text" @click="$emit('update:modelValue', false)">
+          {{ result ? 'Close' : 'Cancel' }}
+        </v-btn>
+        <v-btn
+          v-if="preview && preview.posts && !result"
+          color="error" variant="flat" :loading="busy"
+          @click="confirmOpen = true"
+        >Remove {{ preview.posts }} post(s)</v-btn>
+      </v-card-actions>
+    </v-card>
+    <DestructiveConfirmModal
+      v-if="preview"
+      v-model="confirmOpen"
+      action="delete" kind="other posters' posts" tier="C"
+      :expected-token-override="preview.confirm_token"
+      :projected-counts="{ posts: preview.posts, images: preview.images,
+                           attachments: preview.attachments, grouped: preview.drops }"
+      @confirm="onConfirm"
+    />
+  </v-dialog>
+</template>
+
+<script setup>
+import { ref, watch } from 'vue'
+import { useSourcesStore } from '../../stores/sources.js'
+import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
+
+const props = defineProps({
+  modelValue: { type: Boolean, default: false },
+  source: { type: Object, required: true },
+})
+defineEmits(['update:modelValue'])
+const store = useSourcesStore()
+
+const loading = ref(false)
+const busy = ref(false)
+const error = ref('')
+const preview = ref(null)
+const result = ref(null)
+const confirmOpen = ref(false)
+
+function message(e) {
+  return e?.body?.detail || e?.body?.error || e?.message || String(e)
+}
+
+watch(() => props.modelValue, async (open) => {
+  if (!open) return
+  preview.value = null
+  result.value = null
+  error.value = ''
+  loading.value = true
+  try {
+    preview.value = await store.previewOtherPosters(props.source.id)
+  } catch (e) {
+    error.value = message(e)
+  } finally {
+    loading.value = false
+  }
+})
+
+async function onConfirm(token) {
+  busy.value = true
+  try {
+    result.value = await store.removeOtherPosters(props.source.id, token)
+  } catch (e) {
+    error.value = message(e)
+  } finally {
+    busy.value = false
+  }
+}
+</script>
+
+<style scoped>
+.fc-dim { color: rgb(var(--v-theme-on-surface-variant)); }
+</style>

frontend/src/components/subscriptions/SourceActions.vue

@@ -50,6 +50,17 @@
           library — without re-downloading media
         </v-list-item-subtitle>
       </v-list-item>
+      <v-list-item
+        v-if="hasPosterList && !running"
+        prepend-icon="mdi-account-remove-outline"
+        @click="cleanupOpen = true"
+      >
+        <v-list-item-title>Remove posts from other posters</v-list-item-title>
+        <v-list-item-subtitle>
+          Preview, then delete the posts taken from people outside this
+          source's "Only posts by" list
+        </v-list-item-subtitle>
+      </v-list-item>
       <v-divider v-if="isNative && !running" />
       <v-list-item
         base-color="error"
@@ -59,12 +70,14 @@
         <v-list-item-title>Remove source</v-list-item-title>
       </v-list-item>
     </KebabMenu>
+    <PosterCleanupDialog v-if="hasPosterList" v-model="cleanupOpen" :source="source" />
   </div>
 </template>
 
 <script setup>
-import { computed } from 'vue'
+import { computed, ref } from 'vue'
 import KebabMenu from '../common/KebabMenu.vue'
+import PosterCleanupDialog from './PosterCleanupDialog.vue'
 
 const props = defineProps({
   source: { type: Object, required: true },
@@ -80,6 +93,10 @@ const recapturing = computed(() => !!props.source.backfill_recapture)
 // which those are (`native_ingester`); a copied list here went stale when
 // Discord moved over.
 const isNative = computed(() => !!props.source.native_ingester)
+// #4486: only a Discord source with an "Only posts by" list has other posters.
+const hasPosterList = computed(() => props.source.platform === 'discord'
+  && (props.source.config_overrides?.discord_authors || []).length > 0)
+const cleanupOpen = ref(false)
 </script>
 
 <style scoped>

frontend/src/components/subscriptions/SourceCard.vue

@@ -16,7 +16,7 @@
       <a
         :href="source.url" target="_blank" rel="noopener"
         class="fc-source-card__url" @click.stop
-      >{{ source.url }}</a>
+      >{{ source.display_name || source.url }}</a>
       <v-btn
         icon="mdi-pencil" size="x-small" variant="text"
         @click.stop="$emit('edit', source)"
@@ -39,18 +39,18 @@
       </v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'running'"
-        size="x-small" color="info" variant="tonal" label
+        size="x-small" color="info" variant="flat" label
       >{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
         }}{{ source.backfill_posts
               ? ` · ${source.backfill_posts} posts`
               : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'complete'"
-        size="x-small" color="success" variant="tonal" label
+        size="x-small" color="success" variant="flat" label
       >Backfilled</v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'stalled'"
-        size="x-small" color="warning" variant="tonal" label
+        size="x-small" color="warning" variant="flat" label
       >Stalled</v-chip>
     </div>
 

frontend/src/components/subscriptions/SourceFormDialog.vue

@@ -44,6 +44,17 @@
               v-model="structuredSince" label="Skip posts older than (YYYY-MM-DD)"
               placeholder="2024-01-01" hide-details class="mt-2"
             />
+            <!-- A creator's server is full of other members posting their own
+                 pictures; the source is subscribed to the creator (#4481). -->
+            <v-text-field
+              v-if="platform === 'discord'"
+              v-model="structuredAuthors" label="Only posts by (Discord names or ids)"
+              placeholder="Todding" hint="Comma-separated. Empty takes everyone's."
+              persistent-hint class="mt-2"
+            />
+            <div v-if="platform === 'discord' && authorNames.length" class="text-caption mt-2">
+              {{ authorNames.join(', ') }}
+            </div>
             <p class="text-caption mt-2" style="opacity: 0.75">
               More per-platform fields land here over time. Use Advanced JSON for everything else.
             </p>
@@ -98,6 +109,33 @@ const urlError = ref('')
 const configTab = ref('structured')
 const structuredVideos = ref(true)
 const structuredSince = ref('')
+const structuredAuthors = ref('')
+// Keys the structured view has no field for, carried through its saves so
+// switching tabs never drops what the JSON view set.
+const otherConfig = ref({})
+const STRUCTURED_KEYS = ['videos', 'since', 'discord_authors']
+
+// The extension's poster picker stores ids (#4491) and the name each was
+// picked under, so a list of numbers still says who it means.
+const authorNames = computed(() => {
+  const labels = otherConfig.value.discord_author_labels || {}
+  return splitAuthors(structuredAuthors.value)
+    .map(a => (labels[a] ? `${labels[a]} (${a})` : null))
+    .filter(Boolean)
+})
+
+function splitAuthors(txt) {
+  return (txt || '').split(',').map(s => s.trim()).filter(Boolean)
+}
+
+function takeConfig(co) {
+  structuredVideos.value = co.videos !== false
+  structuredSince.value = co.since ?? ''
+  structuredAuthors.value = Array.isArray(co.discord_authors) ? co.discord_authors.join(', ') : ''
+  otherConfig.value = Object.fromEntries(
+    Object.entries(co).filter(([k]) => !STRUCTURED_KEYS.includes(k)),
+  )
+}
 const jsonText = ref('{}')
 const jsonError = ref('')
 
@@ -105,13 +143,15 @@ const busy = ref(false)
 
 // Sync config_overrides between the two views.
 const config = computed(() => {
-  const out = {}
+  const out = { ...otherConfig.value }
   if (!structuredVideos.value) out.videos = false
   if (structuredSince.value) out.since = structuredSince.value
+  const authors = splitAuthors(structuredAuthors.value)
+  if (authors.length) out.discord_authors = authors
   return out
 })
 
-watch([structuredVideos, structuredSince], () => {
+watch([structuredVideos, structuredSince, structuredAuthors], () => {
   if (configTab.value === 'structured') {
     jsonText.value = JSON.stringify(config.value, null, 2)
     jsonError.value = ''
@@ -128,8 +168,7 @@ watch(jsonText, (txt) => {
     }
     jsonError.value = ''
     // Reflect recognized keys into the structured view.
-    structuredVideos.value = parsed.videos !== false
-    structuredSince.value = parsed.since ?? ''
+    takeConfig(parsed)
   } catch {
     jsonError.value = 'Invalid JSON'
   }
@@ -146,14 +185,13 @@ watch(() => props.modelValue, async (open) => {
     url.value = props.source.url
     enabled.value = props.source.enabled
     const co = props.source.config_overrides || {}
-    structuredVideos.value = co.videos !== false
-    structuredSince.value = co.since ?? ''
+    takeConfig(co)
     jsonText.value = JSON.stringify(co, null, 2)
     artistChoice.value = { id: props.source.artist_id, name: props.source.artist_name }
   } else {
     platform.value = platformsStore.list[0]?.key || 'patreon'
     url.value = ''; enabled.value = true
-    structuredVideos.value = true; structuredSince.value = ''
+    takeConfig({})
     jsonText.value = '{}'
     artistChoice.value = props.initialArtist
       ? { id: props.initialArtist.id, name: props.initialArtist.name }

frontend/src/components/subscriptions/SourceRow.vue

@@ -10,7 +10,7 @@
       <div class="fc-source-row__url-wrap">
         <a :href="source.url" target="_blank" rel="noopener" class="fc-source-row__url"
            @click.stop>
-          {{ source.url }}
+          {{ source.display_name || source.url }}
         </a>
         <!-- Edit sits next to the source identity (operator-requested), not in
              the action cluster where it was easy to fat-finger Remove. -->
@@ -72,7 +72,7 @@
            worked, the content just isn't ours. -->
       <v-chip
         v-else-if="source.error_type === 'tier_limited'"
-        size="x-small" color="info" variant="tonal" label
+        size="x-small" color="info" variant="flat" label
         prepend-icon="mdi-lock-outline"
       >{{ source.tier_gated_count ? `${source.tier_gated_count} gated` : 'No access' }}
         <v-tooltip activator="parent" location="top" max-width="480">
@@ -81,18 +81,18 @@
       </v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'running'"
-        size="x-small" color="info" variant="tonal" label
+        size="x-small" color="info" variant="flat" label
       >{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
         }}{{ source.backfill_posts
               ? ` · ${source.backfill_posts} posts`
               : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'complete'"
-        size="x-small" color="success" variant="tonal" label
+        size="x-small" color="success" variant="flat" label
       >Backfilled</v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'stalled'"
-        size="x-small" color="warning" variant="tonal" label
+        size="x-small" color="warning" variant="flat" label
       >Stalled</v-chip>
       <span v-else class="fc-source-row__zero">0</span>
     </td>

frontend/src/components/subscriptions/SubscriptionsTab.vue

@@ -108,7 +108,7 @@
             v-if="item.singleSource"
             :href="item.singleSource.url" target="_blank" rel="noopener"
             class="fc-subs__sub-url" @click.stop
-          >{{ item.singleSource.url }}</a>
+          >{{ item.singleSource.display_name || item.singleSource.url }}</a>
         </template>
 
         <template #item.platforms="{ item }">
@@ -517,6 +517,7 @@ const filteredGroups = computed(() => {
         g.sources.some(
           (s) =>
             (s.url || '').toLowerCase().includes(q) ||
+            (s.display_name || '').toLowerCase().includes(q) ||
             (s.platform || '').toLowerCase().includes(q),
         ),
     )

frontend/src/stores/backup.js

@@ -8,13 +8,15 @@ export const useBackupStore = defineStore('backup', () => {
 
   const dbRuns = ref([])
   const imagesRuns = ref([])
+  const mediaRuns = ref([])
   const settings = ref(null)
-  const loading = ref({ dbRuns: false, imagesRuns: false, settings: false })
+  const loading = ref({ dbRuns: false, imagesRuns: false, mediaRuns: false, settings: false })
   const lastError = ref(null)
+  const RUNS = { db: dbRuns, images: imagesRuns, media: mediaRuns }
 
   async function loadRuns(kind) {
-    const targetRef = kind === 'db' ? dbRuns : imagesRuns
-    const loadingKey = kind === 'db' ? 'dbRuns' : 'imagesRuns'
+    const targetRef = RUNS[kind]
+    const loadingKey = `${kind}Runs`
     loading.value[loadingKey] = true
     lastError.value = null
     try {
@@ -101,9 +103,47 @@ export const useBackupStore = defineStore('backup', () => {
     }
   }
 
+  // Off-site destination (an S3 bucket). The secret key goes up but never
+  // comes back: the server answers `secret_set` instead.
+  const destination = ref(null)
+
+  async function loadDestination() {
+    destination.value = await api.get('/api/system/backup/destination')
+    return destination.value
+  }
+
+  async function saveDestination(fields) {
+    destination.value = await api.patch('/api/system/backup/destination', { body: fields })
+    return destination.value
+  }
+
+  async function removeDestination() {
+    destination.value = await api.delete('/api/system/backup/destination')
+    return destination.value
+  }
+
+  // Resolves to {ok, message}; an unreachable bucket is a result, not a throw.
+  async function testDestination(fields) {
+    return api.post('/api/system/backup/destination/test', { body: fields })
+  }
+
+  // How much of the library is in the off-site bucket:
+  // {configured, files_total, files_sent, bytes_total, bytes_sent}.
+  const mediaCoverage = ref(null)
+
+  async function loadMediaCoverage() {
+    try {
+      mediaCoverage.value = await api.get('/api/system/backup/media')
+    } catch (e) {
+      lastError.value = e.message
+    }
+  }
+
   return {
-    dbRuns, imagesRuns, settings, loading, lastError,
+    dbRuns, imagesRuns, mediaRuns, mediaCoverage, loadMediaCoverage,
+    settings, loading, lastError,
     loadRuns, triggerBackup, restore, deleteRun, setTag,
     loadSettings, patchSettings,
+    destination, loadDestination, saveDestination, removeDestination, testDestination,
   }
 })

frontend/src/stores/cleanup.js

@@ -12,7 +12,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
     min_width: 0,
     min_height: 0,
     transparency_threshold: 0.9,
-    single_color_threshold: 0.95,
+    single_color_threshold: 0.995,
     single_color_tolerance: 30,
   })
 
@@ -24,7 +24,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
       min_width: s.min_width ?? 0,
       min_height: s.min_height ?? 0,
       transparency_threshold: s.transparency_threshold ?? 0.9,
-      single_color_threshold: s.single_color_threshold ?? 0.95,
+      single_color_threshold: s.single_color_threshold ?? 0.995,
       single_color_tolerance: s.single_color_tolerance ?? 30,
     }
   }

frontend/src/stores/sources.js

@@ -149,6 +149,15 @@ export const useSourcesStore = defineStore('sources', () => {
     return body
   }
 
+  // #4486: a Discord source's posts by people outside its poster list.
+  // The preview touches nothing and hands back the token the remove must echo.
+  async function previewOtherPosters(id) {
+    return api.get(`/api/sources/${id}/discord/other-posters`)
+  }
+  async function removeOtherPosters(id, confirm) {
+    return api.post(`/api/sources/${id}/discord/other-posters/remove`, { body: { confirm } })
+  }
+
   function sourcesByArtistGrouped() {
     // returns [{artist: {id,name,slug}, sources: [...]}, ...]
     const arr = byArtist.value.get(null) ?? []
@@ -179,6 +188,7 @@ export const useSourcesStore = defineStore('sources', () => {
     stopBackfill,
     recoverSource,
     recaptureSource,
+    previewOtherPosters, removeOtherPosters,
     findOrCreateArtist, autocompleteArtist, reassign,
     loadScheduleStatus,
     sourcesByArtistGrouped,

frontend/src/views/CleanupView.vue

@@ -22,13 +22,15 @@
       <h3 class="fc-section__title">Duplicates &amp; leftovers</h3>
       <p class="fc-section__hint">
         Tidy post records, duplicates, locked-preview leftovers and attachments
-        that outlived what they belonged to.
+        that outlived what they belonged to, and import downloads that never
+        made it into the library.
       </p>
       <div class="fc-tile-grid">
         <PostMaintenanceCard />
         <VideoDedupCard />
         <GatedPurgeCard />
         <AttachmentReclaimCard />
+        <LibraryImportCard />
       </div>
     </section>
 
@@ -63,6 +65,7 @@ import PostMaintenanceCard from '../components/settings/PostMaintenanceCard.vue'
 import VideoDedupCard from '../components/settings/VideoDedupCard.vue'
 import GatedPurgeCard from '../components/settings/GatedPurgeCard.vue'
 import AttachmentReclaimCard from '../components/settings/AttachmentReclaimCard.vue'
+import LibraryImportCard from '../components/settings/LibraryImportCard.vue'
 import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
 import DangerZoneCard from '../components/settings/DangerZoneCard.vue'
 </script>

tests/conftest.py

@@ -85,6 +85,29 @@ def db_sync(sync_engine):
             session.rollback()
 
 
+@pytest.fixture
+def celery_conf():
+    """Set Celery config keys for one test: `celery_conf(task_always_eager=True)`.
+    The values they had before the test come back after it.
+
+    Use this, not `monkeypatch.setattr(celery.conf, ...)`. Celery serves config
+    keys through `__getattr__`, so monkeypatch sees no attribute to restore and
+    deletes the key on undo instead; the delete raises, and every later undo of
+    that test is skipped, leaking its patches into the tests after it.
+    """
+    from backend.app.celery_app import celery
+
+    saved: dict[str, object] = {}
+
+    def _set(**values: object) -> None:
+        for key in values:
+            saved.setdefault(key, celery.conf[key])
+        celery.conf.update(values)
+
+    yield _set
+    celery.conf.update(saved)
+
+
 @pytest_asyncio.fixture
 async def app():
     return create_app()

tests/test_api_admin.py

@@ -708,3 +708,22 @@ async def test_trigger_reclaim_attachments_threads_apply(client, monkeypatch):
     )
     assert resp.status_code == 202
     assert calls[0][1] == {"dry_run": False}
+
+
+@pytest.mark.asyncio
+async def test_trigger_import_unimported_defaults_to_preview(client, monkeypatch):
+    """An empty body previews: the apply removes files the library rejects."""
+    from backend.app.tasks import admin as admin_tasks
+
+    calls = []
+    monkeypatch.setattr(
+        admin_tasks.import_unimported_files_task, "delay", _fake_delay(calls)
+    )
+    resp = await client.post("/api/admin/maintenance/import-unimported", json={})
+    assert resp.status_code == 202
+    assert calls[0][1] == {"dry_run": True}
+
+    resp = await client.post(
+        "/api/admin/maintenance/import-unimported", json={"dry_run": False},
+    )
+    assert calls[1][1] == {"dry_run": False}

tests/test_api_cleanup.py

@@ -11,15 +11,14 @@ import pytest
 from PIL import Image
 from sqlalchemy import func, select
 
-from backend.app.celery_app import celery
 from backend.app.models import ImageRecord, LibraryAuditRun
 
 pytestmark = pytest.mark.integration
 
 
 @pytest.fixture(autouse=True)
-def disable_celery_eager(monkeypatch):
-    monkeypatch.setattr(celery.conf, "task_always_eager", False)
+def disable_celery_eager(celery_conf):
+    celery_conf(task_always_eager=False)
 
 
 def _sha256_min_dim_token(min_w: int, min_h: int) -> str:

tests/test_api_system_backup.py

@@ -232,6 +232,27 @@ async def test_restore_rejects_non_ok_status(client, db):
     assert body["error"] == "not_restorable"
 
 
+@pytest.mark.asyncio
+async def test_an_off_site_dump_is_not_restored_from_this_screen(client, db):
+    db.add(BackupRun(
+        kind="db", status="ok", tag=None, triggered_by="nightly",
+        started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
+        manifest={"offsite": {"endpoint": "http://archives.example:8081",
+                              "bucket": "curator", "key": "database/x.dump"}},
+    ))
+    await db.commit()
+    run = (await (await client.get("/api/system/backup/runs?kind=db&limit=1")).get_json())[
+        "runs"][0]
+    assert run["offsite"]["bucket"] == "curator"
+
+    resp = await client.post(
+        f"/api/system/backup/runs/{run['id']}/restore",
+        json={"confirm": f"restore-db-{run['id']}"},
+    )
+    assert resp.status_code == 400
+    assert (await resp.get_json())["error"] == "not_restorable"
+
+
 # --- DELETE /runs/<id> ----------------------------------------------
 
 
@@ -322,3 +343,156 @@ async def test_patch_settings_rejects_keep_n_below_min(client):
         json={"backup_db_keep_last_n": 0},
     )
     assert resp.status_code == 400
+
+
+# --- /destination ---------------------------------------------------
+
+
+@pytest.fixture
+def _sealing_key(tmp_path, monkeypatch):
+    from backend.app.services import backup_destination
+    monkeypatch.setattr(backup_destination, "KEY_PATH", tmp_path / "credential_key.b64")
+
+
+_DEST = {
+    "endpoint": "http://archives.example:8081/",
+    "bucket": "curator",
+    "access_key": "AKEXAMPLE",
+    "region": "",
+    "secret_key": "s3cr3t-example",
+}
+
+
+@pytest.mark.asyncio
+async def test_destination_saves_and_never_returns_the_secret(client, db_sync, _sealing_key):
+    from backend.app.models import ImportSettings
+
+    resp = await client.patch("/api/system/backup/destination", json=_DEST)
+    assert resp.status_code == 200
+    body = await resp.get_json()
+    assert body == {
+        "endpoint": "http://archives.example:8081", "bucket": "curator",
+        "access_key": "AKEXAMPLE", "region": "us-east-1",
+        "secret_set": True, "configured": True, "upload_limit_mbps": 200,
+    }
+    assert "s3cr3t-example" not in str(await (await client.get(
+        "/api/system/backup/destination")).get_json())
+    row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
+    db_sync.refresh(row)
+    assert row.backup_s3_secret_sealed and "s3cr3t-example" not in row.backup_s3_secret_sealed
+
+    from backend.app.services import backup_destination
+    assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
+
+
+@pytest.mark.asyncio
+async def test_destination_blank_secret_keeps_the_saved_one(client, db_sync, _sealing_key):
+    from backend.app.models import ImportSettings
+    from backend.app.services import backup_destination
+
+    await client.patch("/api/system/backup/destination", json=_DEST)
+    resp = await client.patch(
+        "/api/system/backup/destination", json={**_DEST, "bucket": "other", "secret_key": ""},
+    )
+    assert (await resp.get_json())["bucket"] == "other"
+    row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
+    db_sync.refresh(row)
+    assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("patch, word", [
+    ({"endpoint": "archives.example:8081"}, "http://"),
+    ({"endpoint": "ftp://archives.example"}, "http://"),
+    ({"bucket": "curator/sub"}, "bucket"),
+    ({"access_key": 5}, "access_key"),
+])
+async def test_destination_rejects_what_cant_be_an_s3_target(client, _sealing_key, patch, word):
+    resp = await client.patch("/api/system/backup/destination", json={**_DEST, **patch})
+    assert resp.status_code == 400
+    assert word in (await resp.get_json())["detail"]
+
+
+@pytest.mark.asyncio
+async def test_destination_remove_forgets_everything(client, _sealing_key):
+    await client.patch("/api/system/backup/destination", json=_DEST)
+    resp = await client.delete("/api/system/backup/destination")
+    assert await resp.get_json() == {
+        "endpoint": "", "bucket": "", "access_key": "", "region": "us-east-1",
+        "secret_set": False, "configured": False, "upload_limit_mbps": 200,
+    }
+
+
+@pytest.mark.asyncio
+async def test_destination_test_uses_the_form_and_the_saved_secret(
+    client, _sealing_key, monkeypatch,
+):
+    from backend.app.services import backup_destination
+
+    seen = []
+
+    def fake_check(target):
+        seen.append(target)
+        return {"ok": True, "message": "Reached it."}
+
+    monkeypatch.setattr(backup_destination, "check_target", fake_check)
+    await client.patch("/api/system/backup/destination", json=_DEST)
+    resp = await client.post(
+        "/api/system/backup/destination/test",
+        json={**_DEST, "bucket": "typed-not-saved", "secret_key": ""},
+    )
+    assert await resp.get_json() == {"ok": True, "message": "Reached it."}
+    assert seen[0].bucket == "typed-not-saved"
+    assert seen[0].secret_key == "s3cr3t-example"
+    assert seen[0].endpoint == "http://archives.example:8081"
+
+
+@pytest.mark.asyncio
+async def test_destination_test_names_what_is_missing(client, _sealing_key):
+    await client.delete("/api/system/backup/destination")
+    resp = await client.post(
+        "/api/system/backup/destination/test",
+        json={"endpoint": "http://archives.example:8081", "bucket": "curator"},
+    )
+    body = await resp.get_json()
+    assert body["ok"] is False
+    assert body["message"] == "Not tested: fill in the access key, secret key."
+
+
+@pytest.mark.asyncio
+async def test_destination_test_reports_an_unreachable_bucket_as_a_result(
+    client, _sealing_key, monkeypatch,
+):
+    from backend.app.services import backup_destination
+    from backend.app.services.backup_s3 import S3Error
+
+    def refuse(target, **kw):
+        raise S3Error("Couldn't reach http://archives.example:8081: nothing is listening there.")
+
+    monkeypatch.setattr(backup_destination, "list_objects", refuse)
+    resp = await client.post("/api/system/backup/destination/test", json=_DEST)
+    assert resp.status_code == 200
+    assert await resp.get_json() == {
+        "ok": False,
+        "message": "Couldn't reach http://archives.example:8081: nothing is listening there.",
+    }
+
+
+@pytest.mark.asyncio
+async def test_the_upload_limit_saves_and_a_missing_one_keeps_it(client, _sealing_key):
+    resp = await client.patch(
+        "/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": 50},
+    )
+    assert (await resp.get_json())["upload_limit_mbps"] == 50
+    resp = await client.patch("/api/system/backup/destination", json=_DEST)
+    assert (await resp.get_json())["upload_limit_mbps"] == 50
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("limit", [-1, 2.5, "fast", True])
+async def test_an_upload_limit_that_isnt_a_whole_number_is_refused(client, _sealing_key, limit):
+    resp = await client.patch(
+        "/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": limit},
+    )
+    assert resp.status_code == 400
+    assert "upload_limit_mbps" in (await resp.get_json())["detail"]

tests/test_api_workers.py

@@ -13,12 +13,25 @@ from sqlalchemy import select
 from backend.app.api import workers as workers_api
 from backend.app.models import WorkerLane
 from backend.app.services import worker_control as wc
+from backend.app.services import worker_lanes as wl
 from backend.app.services.worker_lanes import LANES, SWEEP_PERIOD_SECONDS
 from tests.doubles import RecordingSession
 
 pytestmark = pytest.mark.integration
 
 
+@pytest.fixture(autouse=True)
+def a_known_machine(monkeypatch):
+    """A 16-core, 64 GiB container, whatever runs the suite.
+
+    Each lane's ceiling is derived from the container's CPU quota and memory
+    limit, so without this the caps these tests may set depend on the CI
+    runner's size, and a smaller runner refuses them as above the ceiling.
+    """
+    monkeypatch.setattr(wl, "container_cpu_count", lambda: 16)
+    monkeypatch.setattr(wl, "container_memory_bytes", lambda: 64 * wl.GIB)
+
+
 @pytest_asyncio.fixture
 async def no_live_workers(monkeypatch):
     """Nothing is running — which is the CI lane's actual truth, asserted

tests/test_audits_single_color.py

@@ -40,3 +40,48 @@ def test_single_color_evaluate_handles_rgba_input():
     # Alpha channel should be ignored — only RGB matters for the rule.
     im = Image.new("RGBA", (50, 50), (100, 100, 100, 128))
     assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True
+
+
+# -- line art is a drawing, not a blank (#4483) ---------------------------------
+
+_T = 0.995  # the default since alembic 0116
+_TOL = 30
+
+
+def _doodle(size=(1500, 2000), lines=12):
+    """Pencil line art on white: a few thin dark strokes plus faint
+    construction lines — a couple of percent ink, the rest paper."""
+    from PIL import ImageDraw
+
+    im = Image.new("RGB", size, (255, 255, 255))
+    draw = ImageDraw.Draw(im)
+    w, h = size
+    for i in range(lines):
+        x = (i + 1) * w // (lines + 1)
+        draw.line([(x, 0), (w - x, h)], fill=(70, 60, 60), width=3)
+        draw.line([(0, (i + 1) * h // (lines + 1)), (w, h - x)], fill=(235, 235, 235), width=2)
+    return im
+
+
+def test_line_art_on_white_is_not_single_color():
+    """Five of Todding's Discord doodles were skipped as blank: a smoothing
+    64px thumbnail blended the strokes into the paper."""
+    assert single_color.evaluate(_doodle(), threshold=_T, tolerance=_TOL) is False
+
+
+def test_a_blank_page_with_a_small_mark_is_still_single_color():
+    from PIL import ImageDraw
+
+    im = Image.new("RGB", (1000, 1000), (250, 250, 250))
+    ImageDraw.Draw(im).rectangle([480, 480, 520, 520], fill=(40, 40, 40))
+    assert single_color.evaluate(im, threshold=_T, tolerance=_TOL) is True
+
+
+def test_a_noisy_solid_fill_is_still_single_color():
+    """JPEG noise on a placeholder stays inside the tolerance."""
+    im = Image.new("RGB", (600, 600), (30, 90, 160))
+    for i in range(0, 600 * 600, 7):
+        x, y = i % 600, i // 600
+        d = (i % 21) - 10
+        im.putpixel((x, y), (30 + d, 90 + d, 160 + d))
+    assert single_color.evaluate(im, threshold=_T, tolerance=_TOL) is True

tests/test_backup_media.py

@@ -0,0 +1,264 @@
+"""The off-site copy of the library's files (services/backup_media.py and the
+backup_media_task chunk loop): each image and attachment goes to the bucket
+once, by sha256, the work list is what isn't there yet, and a file that can't
+be sent is reported and tried again rather than stopping the run."""
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import select
+
+from backend.app.models import (
+    Artist,
+    BackupBlob,
+    BackupRun,
+    ImageRecord,
+    Post,
+    PostAttachment,
+)
+from backend.app.services import backup_media
+from backend.app.services.backup_s3 import ListPage, S3Error, S3Target, StoredObject
+
+pytestmark = pytest.mark.integration
+
+TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK")
+TID = "http://archives.example:8081/curator"
+
+
+def _sha(c: str) -> str:
+    return c * 64
+
+
+def _image(db_sync, tmp_path, c, body=b"img", ext=".jpg"):
+    path = tmp_path / f"{c}{ext}"
+    path.write_bytes(body)
+    db_sync.add(ImageRecord(path=str(path), sha256=_sha(c), size_bytes=len(body),
+                            mime="image/jpeg", origin="imported_filesystem"))
+    return path
+
+
+def _attachment(db_sync, tmp_path, c, body=b"zip", name="set.zip"):
+    path = tmp_path / f"{c}-{name}"
+    path.write_bytes(body)
+    db_sync.add(PostAttachment(sha256=_sha(c), path=str(path), original_filename=name,
+                               ext=".zip", size_bytes=len(body)))
+    return path
+
+
+# ── keys and the work list ──────────────────────────────────────────────────
+
+
+def test_keys_are_content_addressed_under_media():
+    sha = "abcd" + "0" * 60
+    assert backup_media.blob_key(sha, "/images/x/Pic.JPG") == f"media/ab/cd/{sha}.jpg"
+    assert backup_media.blob_key(sha, "/images/x/noext") == f"media/ab/cd/{sha}"
+    assert backup_media.blob_key(sha, "/images/x/a.we ird") == f"media/ab/cd/{sha}"
+
+
+def test_the_work_list_is_every_file_once_that_the_bucket_lacks(db_sync, tmp_path):
+    _image(db_sync, tmp_path, "a")
+    _image(db_sync, tmp_path, "b")
+    _attachment(db_sync, tmp_path, "c")
+    # The same archive held by a post as well is one file to send.
+    artist = Artist(name="Bea", slug="bea")
+    db_sync.add(artist)
+    db_sync.flush()
+    post = Post(artist_id=artist.id, external_post_id="p1")
+    db_sync.add(post)
+    db_sync.flush()
+    db_sync.add(PostAttachment(post_id=post.id, sha256=_sha("c"),
+                               path=str(tmp_path / "c-copy.zip"),
+                               original_filename="set.zip", ext=".zip", size_bytes=3))
+    db_sync.add(BackupBlob(target=TID, sha256=_sha("b"), key="media/bb", size_bytes=3))
+    # A row for another bucket doesn't count for this one.
+    db_sync.add(BackupBlob(target="http://other/bucket", sha256=_sha("a"), key="k",
+                           size_bytes=3))
+    db_sync.commit()
+
+    shas = [b.sha256 for b in backup_media.pending(db_sync, TID)]
+    assert shas == [_sha("a"), _sha("c")]
+    assert [b.sha256 for b in backup_media.pending(db_sync, TID, after=_sha("a"))] == [_sha("c")]
+
+    cov = backup_media.coverage(db_sync, TID)
+    assert cov == {"files_total": 3, "bytes_total": 9, "files_sent": 1, "bytes_sent": 3}
+
+
+# ── sending one file ────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def puts(monkeypatch):
+    sent = []
+    monkeypatch.setattr(backup_media, "put_file",
+                        lambda t, key, path, **kw: sent.append(("whole", key, kw)))
+    monkeypatch.setattr(backup_media, "put_file_in_parts",
+                        lambda t, key, path, **kw: sent.append(("parts", key, kw)))
+    return sent
+
+
+def test_a_file_is_signed_with_its_recorded_sha(db_sync, tmp_path, puts):
+    path = _image(db_sync, tmp_path, "a")
+    blob = backup_media.Blob(_sha("a"), str(path), 3, "image/jpeg")
+    key = backup_media.send(TARGET, blob, tmp_path, None)
+    assert key == f"media/aa/aa/{_sha('a')}.jpg"
+    (how, sent_key, kw) = puts[0]
+    assert (how, sent_key) == ("whole", key)
+    assert kw["payload_sha256"] == _sha("a") and kw["content_type"] == "image/jpeg"
+
+
+def test_a_large_file_goes_in_parts(db_sync, tmp_path, puts, monkeypatch):
+    monkeypatch.setattr(backup_media, "MULTIPART_OVER", 2)
+    path = _attachment(db_sync, tmp_path, "c")
+    backup_media.send(TARGET, backup_media.Blob(_sha("c"), str(path), 3, None), tmp_path, None)
+    assert puts[0][0] == "parts"
+    assert puts[0][2]["sha256"] == _sha("c")
+
+
+def test_a_missing_or_changed_file_is_refused_before_sending(db_sync, tmp_path, puts):
+    gone = backup_media.Blob(_sha("a"), str(tmp_path / "gone.jpg"), 3, None)
+    with pytest.raises(S3Error, match="not on disk"):
+        backup_media.send(TARGET, gone, tmp_path, None)
+    path = _image(db_sync, tmp_path, "b", body=b"longer now")
+    changed = backup_media.Blob(_sha("b"), str(path), 3, None)
+    with pytest.raises(S3Error, match="when imported"):
+        backup_media.send(TARGET, changed, tmp_path, None)
+    assert puts == []
+
+
+def test_a_refusal_for_an_object_already_there_counts_as_sent(db_sync, tmp_path, monkeypatch):
+    def refused(*a, **kw):
+        raise S3Error("HTTP 403 AccessDenied: append-only.")
+    monkeypatch.setattr(backup_media, "put_file", refused)
+    monkeypatch.setattr(backup_media, "object_present", lambda t, key, size: True)
+    path = _image(db_sync, tmp_path, "a")
+    key = backup_media.send(TARGET, backup_media.Blob(_sha("a"), str(path), 3, None),
+                            tmp_path, None)
+    assert key.startswith("media/aa/aa/")
+
+
+# ── the run ─────────────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def run_env(monkeypatch, tmp_path):
+    monkeypatch.setattr("backend.app.tasks.backup.IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(
+        "backend.app.services.backup_destination.saved_target", lambda settings: TARGET,
+    )
+    queued = []
+    from backend.app.tasks import backup as tasks
+    monkeypatch.setattr(tasks.backup_media_task, "delay", lambda **kw: queued.append(kw))
+    return queued
+
+
+def _send_all_but(bad_sha):
+    def send(target, blob, images_root, pacer):
+        if blob.sha256 == bad_sha:
+            raise S3Error("HTTP 503: busy.")
+        return backup_media.blob_key(blob.sha256, blob.path)
+    return send
+
+
+def test_a_run_sends_what_is_missing_and_reports_what_failed(
+    db_sync, tmp_path, run_env, monkeypatch,
+):
+    from backend.app.tasks.backup import backup_media_task
+    for c in "abc":
+        _image(db_sync, tmp_path, c)
+    db_sync.commit()
+    monkeypatch.setattr(backup_media, "send", _send_all_but(_sha("b")))
+
+    result = backup_media_task(triggered_by="manual")
+    assert result["continues"] is False
+    row = db_sync.get(BackupRun, result["backup_run_id"])
+    db_sync.refresh(row)
+    assert (row.kind, row.status, row.size_bytes) == ("media", "ok", 6)
+    assert row.manifest["sent"] == 2 and row.manifest["failed"] == 1
+    assert row.manifest["offsite"]["bucket"] == "curator"
+    assert "1 file(s) weren't sent" in row.error
+    assert row.manifest["failures"][0]["error"] == "HTTP 503: busy."
+    sent = set(db_sync.execute(select(BackupBlob.sha256)).scalars())
+    assert sent == {_sha("a"), _sha("c")}
+
+    # The next run tries only the file that failed.
+    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
+    again = backup_media_task(triggered_by="manual")
+    row = db_sync.get(BackupRun, again["backup_run_id"])
+    db_sync.refresh(row)
+    assert row.manifest["sent"] == 1 and row.error is None
+
+
+def test_a_run_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch):
+    from backend.app.tasks import backup as tasks
+    for c in "ab":
+        _image(db_sync, tmp_path, c)
+    db_sync.commit()
+    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
+    monkeypatch.setattr(tasks, "MEDIA_CHUNK_SECONDS", -1)
+
+    result = tasks.backup_media_task(triggered_by="manual")
+    assert result["continues"] is True
+    assert run_env == [{"run_id": result["backup_run_id"]}]
+    row = db_sync.get(BackupRun, result["backup_run_id"])
+    db_sync.refresh(row)
+    assert row.status == "running" and row.last_progress_at is not None
+
+
+def test_a_second_run_waits_for_the_one_going(db_sync, run_env):
+    from backend.app.tasks.backup import backup_media_task
+    now = datetime.now(UTC)
+    db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly",
+                          started_at=now, manifest={}))
+    db_sync.commit()
+    assert "already running" in backup_media_task(triggered_by="manual")["skipped"]
+
+
+def test_without_a_destination_the_run_fails_with_why(db_sync, monkeypatch):
+    from backend.app.tasks.backup import backup_media_task
+    monkeypatch.setattr(
+        "backend.app.services.backup_destination.saved_target", lambda settings: None,
+    )
+    with pytest.raises(S3Error):
+        backup_media_task(triggered_by="manual")
+    row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "media")).scalar_one()
+    assert row.status == "error" and "No off-site destination" in row.error
+
+
+def test_a_long_media_run_is_judged_stalled_by_its_heartbeat(db_sync):
+    from backend.app.tasks.maintenance import recover_stalled_backup_runs
+    long_ago = datetime.now(UTC) - timedelta(hours=20)
+    alive = BackupRun(kind="media", status="running", triggered_by="nightly",
+                      started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={})
+    silent = BackupRun(kind="media", status="running", triggered_by="nightly",
+                       started_at=long_ago, last_progress_at=long_ago, manifest={})
+    db_sync.add_all([alive, silent])
+    db_sync.commit()
+    assert recover_stalled_backup_runs() == 1
+    db_sync.refresh(alive)
+    db_sync.refresh(silent)
+    assert (alive.status, silent.status) == ("running", "error")
+
+
+# ── reconcile ───────────────────────────────────────────────────────────────
+
+
+def test_reconcile_resends_what_the_bucket_lost_and_adopts_what_it_has(
+    db_sync, tmp_path, monkeypatch,
+):
+    _image(db_sync, tmp_path, "a")            # row, but the bucket lost it
+    _image(db_sync, tmp_path, "b")            # in the bucket, no row
+    _image(db_sync, tmp_path, "c")            # neither
+    key_a = backup_media.blob_key(_sha("a"), "x.jpg")
+    key_b = backup_media.blob_key(_sha("b"), "x.jpg")
+    db_sync.add(BackupBlob(target=TID, sha256=_sha("a"), key=key_a, size_bytes=3))
+    db_sync.commit()
+    pages = iter([
+        ListPage([StoredObject("media/zz/other", 1, "")], "next"),
+        ListPage([StoredObject(key_b, 3, "")], None),
+    ])
+    monkeypatch.setattr(backup_media, "list_objects", lambda *a, **kw: next(pages))
+    monkeypatch.setattr(backup_media.time, "sleep", lambda s: None)
+
+    assert backup_media.reconcile(db_sync, TARGET) == {
+        "listed": 2, "dropped": 1, "adopted": 1,
+    }
+    assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}

tests/test_backup_s3.py

@@ -0,0 +1,363 @@
+"""The backup destination's S3 client (services/backup_s3.py): signatures match
+AWS's published Signature Version 4 examples, and the calls send and read what
+S3 expects, over a real HTTP connection to a local stand-in server."""
+import hashlib
+import socket
+import threading
+from datetime import UTC, datetime
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+from backend.app.services import backup_s3 as s3
+
+# AWS's example credentials and moment ("Signature Calculations for the
+# Authorization Header", Amazon S3 API reference).
+KEY_ID = "AKIAIOSFODNN7EXAMPLE"
+SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+AT = datetime(2013, 5, 24, tzinfo=UTC)
+
+
+def _signature(headers):
+    return headers["Authorization"].rsplit("Signature=", 1)[1]
+
+
+def test_aws_example_get_object():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt",
+                {"Range": "bytes=0-9"}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"]
+    assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h[
+        "Authorization"]
+    assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
+
+
+def test_aws_example_list_objects():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
+
+
+def test_a_changed_secret_changes_the_signature():
+    url = "https://examplebucket.s3.amazonaws.com/test.txt"
+    a = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    b = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT)
+    assert _signature(a) != _signature(b)
+
+
+def test_paths_are_path_style_and_keys_are_escaped():
+    t = s3.S3Target("http://archives.example:8081/s3", "curator", KEY_ID, SECRET)
+    assert t.url("media/ab/cd/abcd.jpg") == "http://archives.example:8081/s3/curator/media/ab/cd/abcd.jpg"
+    assert t.url() == "http://archives.example:8081/s3/curator/"
+    assert t.url("a b+c.jpg") == "http://archives.example:8081/s3/curator/a%20b%2Bc.jpg"
+
+
+# ── Calls, against a local stand-in ─────────────────────────────────────────
+
+
+class _Server:
+    """A local HTTP server that records each request and answers with
+    `reply(handler) -> (status, body)` or `(status, body, headers)`."""
+
+    def __init__(self, reply):
+        self.seen = []
+        outer = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def _serve(self):
+                length = int(self.headers.get("Content-Length") or 0)
+                outer.seen.append({
+                    "method": self.command, "path": self.path,
+                    "headers": {k.lower(): v for k, v in self.headers.items()},
+                    "body": self.rfile.read(length) if length else b"",
+                })
+                status, body, *extra = reply(self)
+                self.send_response(status)
+                for name, value in (extra[0] if extra else {}).items():
+                    self.send_header(name, value)
+                self.send_header("Content-Length", str(len(body)))
+                self.end_headers()
+                self.wfile.write(body)
+
+            do_GET = do_PUT = do_POST = do_DELETE = _serve
+
+            def log_message(self, *args):
+                pass
+
+        self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+
+    def __enter__(self):
+        self.thread.start()
+        return self
+
+    def __exit__(self, *exc):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def target(self, bucket="curator"):
+        host, port = self.httpd.server_address
+        return s3.S3Target(f"http://{host}:{port}", bucket, KEY_ID, SECRET)
+
+
+def test_put_streams_the_file_signed(tmp_path):
+    f = tmp_path / "abcd.jpg"
+    f.write_bytes(b"x" * 5000)
+    with _Server(lambda h: (200, b"")) as srv:
+        s3.put_file(srv.target(), "media/ab/cd/abcd.jpg", f, content_type="image/jpeg")
+    req = srv.seen[0]
+    assert req["method"] == "PUT"
+    assert req["path"] == "/curator/media/ab/cd/abcd.jpg"
+    assert req["body"] == b"x" * 5000
+    assert req["headers"]["x-amz-content-sha256"] == s3.file_sha256(f)
+    assert req["headers"]["content-type"] == "image/jpeg"
+    assert req["headers"]["authorization"].startswith(
+        "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/")
+
+
+def test_put_signs_with_a_known_sha_without_rereading(tmp_path, monkeypatch):
+    f = tmp_path / "b.bin"
+    f.write_bytes(b"y")
+    monkeypatch.setattr(s3, "file_sha256", lambda p: pytest.fail("read the file twice"))
+    with _Server(lambda h: (200, b"")) as srv:
+        s3.put_file(srv.target(), "b.bin", f, payload_sha256="a" * 64)
+    assert srv.seen[0]["headers"]["x-amz-content-sha256"] == "a" * 64
+
+
+def test_a_refusal_says_why(tmp_path):
+    f = tmp_path / "b.bin"
+    f.write_bytes(b"x")
+    body = b"<Error><Code>SignatureDoesNotMatch</Code><Message>no</Message></Error>"
+    with _Server(lambda h: (403, body)) as srv, pytest.raises(
+        s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong",
+    ):
+        s3.put_file(srv.target(), "b.bin", f)
+
+
+def test_a_web_server_at_the_address_is_named_as_not_s3():
+    """An address without the S3 port lands on whatever serves the default
+    port, and its plain 404 says nothing about S3."""
+    page = b"<html><body><h1>404 Not Found</h1></body></html>"
+    with _Server(lambda h: (404, page)) as srv, pytest.raises(
+        s3.S3Error,
+        match=r"HTTP 404: the address answered, but not as S3 — check the scheme and the port",
+    ):
+        s3.list_objects(srv.target())
+
+
+def test_a_server_error_without_a_body_is_not_blamed_on_the_port():
+    with _Server(lambda h: (503, b"")) as srv, pytest.raises(
+        s3.S3Error, match="HTTP 503: Service Unavailable",
+    ) as err:
+        s3.list_objects(srv.target())
+    assert "port" not in str(err.value)
+
+
+def test_nothing_listening_says_check_the_port():
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        port = sock.getsockname()[1]
+    target = s3.S3Target(f"http://127.0.0.1:{port}", "curator", KEY_ID, SECRET)
+    with pytest.raises(s3.S3Error, match="nothing is listening there"):
+        s3.list_objects(target)
+
+
+def test_list_reads_contents_and_the_next_page_token():
+    xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>curator</Name><KeyCount>2</KeyCount><IsTruncated>true</IsTruncated>
+  <NextContinuationToken>tok-2</NextContinuationToken>
+  <Contents><Key>database/fabledcurator-20261105T030000Z.dump</Key><Size>1200</Size>
+    <LastModified>2026-11-05T03:00:01.000Z</LastModified></Contents>
+  <Contents><Key>database/fabledcurator-20261106T030000Z.dump</Key><Size>1300</Size>
+    <LastModified>2026-11-06T03:00:01.000Z</LastModified></Contents>
+</ListBucketResult>"""
+    with _Server(lambda h: (200, xml)) as srv:
+        page = s3.list_objects(srv.target(), "database/", continuation="tok-1")
+    path = srv.seen[0]["path"]
+    assert path.startswith("/curator/?")
+    assert "list-type=2" in path and "prefix=database%2F" in path
+    assert "continuation-token=tok-1" in path
+    assert page.next_token == "tok-2"
+    assert page.objects == [
+        s3.StoredObject("database/fabledcurator-20261105T030000Z.dump", 1200,
+                        "2026-11-05T03:00:01.000Z"),
+        s3.StoredObject("database/fabledcurator-20261106T030000Z.dump", 1300,
+                        "2026-11-06T03:00:01.000Z"),
+    ]
+
+
+def test_a_listing_that_isnt_xml_is_named():
+    with _Server(lambda h: (200, b"hello")) as srv, pytest.raises(
+        s3.S3Error, match="isn't S3's XML",
+    ):
+        s3.list_objects(srv.target())
+
+
+def test_get_streams_the_object_to_disk(tmp_path):
+    with _Server(lambda h: (200, b"y" * 3000)) as srv:
+        out = tmp_path / "b.dump"
+        s3.get_file(srv.target(), "database/b.dump", out)
+    assert srv.seen[0]["path"] == "/curator/database/b.dump"
+    assert srv.seen[0]["headers"]["x-amz-content-sha256"] == s3.EMPTY_SHA256
+    assert out.read_bytes() == b"y" * 3000
+    assert not (tmp_path / "b.dump.partial").exists()
+
+
+def test_a_refused_get_says_so_and_writes_nothing(tmp_path):
+    body = b"<Error><Code>AccessDenied</Code></Error>"
+    out = tmp_path / "b.dump"
+    with _Server(lambda h: (403, body)) as srv, pytest.raises(s3.S3Error, match="b.dump"):
+        s3.get_file(srv.target(), "b.dump", out)
+    assert not out.exists()
+    assert not (tmp_path / "b.dump.partial").exists()
+
+
+# ── Multipart ───────────────────────────────────────────────────────────────
+
+_STARTED = b"<InitiateMultipartUploadResult><UploadId>up1</UploadId></InitiateMultipartUploadResult>"
+
+
+def _bucket(parts_reply=None, complete=(200, b"<CompleteMultipartUploadResult/>")):
+    """A stand-in bucket for one multipart upload: starts it, takes parts (each
+    answered by `parts_reply(n)`, an ETag by default) and completes it."""
+    def reply(h):
+        if h.command == "POST" and h.path.endswith("?uploads"):
+            return 200, _STARTED
+        if h.command == "PUT":
+            n = int(h.path.split("partNumber=")[1].split("&")[0])
+            return parts_reply(n) if parts_reply else (200, b"", {"ETag": f'"e{n}"'})
+        if h.command == "POST":
+            return complete
+        return 204, b""
+    return reply
+
+
+def test_an_upload_sends_its_parts_then_joins_them_in_order():
+    with _Server(_bucket()) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.add(b"first,")
+        up.add(b"second")
+        up.complete()
+    start, one, two, done = srv.seen
+    assert (start["method"], start["path"]) == ("POST", "/curator/database/x.dump?uploads")
+    assert one["path"] == "/curator/database/x.dump?partNumber=1&uploadId=up1"
+    assert two["body"] == b"second"
+    assert two["headers"]["x-amz-content-sha256"] == hashlib.sha256(b"second").hexdigest()
+    assert done["path"] == "/curator/database/x.dump?uploadId=up1"
+    assert done["body"].index(b'"e1"') < done["body"].index(b'"e2"')
+    assert (up.parts, up.size) == (2, len(b"first,second"))
+
+
+def test_a_refused_start_says_why():
+    with _Server(lambda h: (403, b"<Error><Code>AccessDenied</Code></Error>")) as srv:
+        with pytest.raises(s3.S3Error, match="the key isn't allowed"):
+            s3.Upload(srv.target(), "database/x.dump")
+
+
+def test_a_completion_refused_under_a_200_is_a_failure():
+    error = (200, b"<Error><Code>InternalError</Code><Message>disk full</Message></Error>")
+    with _Server(_bucket(complete=error)) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.add(b"data")
+        with pytest.raises(s3.S3Error, match="Finishing the upload"):
+            up.complete()
+
+
+def test_nothing_sent_is_not_completed():
+    with _Server(_bucket()) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        with pytest.raises(s3.S3Error, match="Nothing was sent"):
+            up.complete()
+
+
+def test_a_busy_bucket_gets_the_part_again_after_a_rest(monkeypatch):
+    rests = []
+    monkeypatch.setattr(s3.time, "sleep", rests.append)
+    answers = iter([(503, b"", {"Retry-After": "7"}), (200, b"", {"ETag": '"e1"'})])
+    with _Server(_bucket(parts_reply=lambda n: next(answers))) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.add(b"data")
+    assert rests == [7]
+    assert up.parts == 1
+
+
+def test_a_part_refused_outright_is_not_sent_again(monkeypatch):
+    monkeypatch.setattr(s3.time, "sleep", lambda s: pytest.fail("rested on a refusal"))
+    refused = (403, b"<Error><Code>AccessDenied</Code></Error>")
+    with _Server(_bucket(parts_reply=lambda n: refused)) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        with pytest.raises(s3.S3Error, match="Part 1"):
+            up.add(b"data")
+
+
+def test_a_part_that_never_lands_gives_up_after_the_rests(monkeypatch):
+    rests = []
+    monkeypatch.setattr(s3.time, "sleep", rests.append)
+    with _Server(_bucket(parts_reply=lambda n: (503, b""))) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        with pytest.raises(s3.S3Error, match="Part 1"):
+            up.add(b"data")
+    assert rests == list(s3.PART_RETRY_WAITS)
+
+
+def test_abort_asks_the_bucket_to_drop_the_parts_and_never_raises():
+    with _Server(_bucket()) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.abort()
+        assert srv.seen[-1]["method"] == "DELETE"
+        assert srv.seen[-1]["path"] == "/curator/database/x.dump?uploadId=up1"
+    up.abort()  # the server is gone now: still no exception
+
+
+def test_the_pacer_holds_bytes_to_the_limit(monkeypatch):
+    clock = [0.0]
+    monkeypatch.setattr(s3.time, "monotonic", lambda: clock[0])
+    monkeypatch.setattr(s3.time, "sleep", lambda secs: clock.__setitem__(0, clock[0] + secs))
+    pacer = s3.Pacer(8)                     # 8 Mbit/s = 1,000,000 bytes/s
+    data = b"x" * (3 * s3.CHUNK)
+    assert b"".join(pacer.chunks(data)) == data
+    assert clock[0] == pytest.approx(len(data) / 1_000_000)
+
+
+def test_no_limit_never_waits(monkeypatch):
+    monkeypatch.setattr(s3.time, "sleep", lambda secs: pytest.fail("waited without a limit"))
+    assert b"".join(s3.Pacer(0).chunks(b"y" * (2 * s3.CHUNK + 1))) == b"y" * (2 * s3.CHUNK + 1)
+
+
+def test_a_large_file_is_sent_in_parts_and_checked_against_its_sha(tmp_path, monkeypatch):
+    monkeypatch.setattr(s3, "PART_SIZE", 4)
+    f = tmp_path / "big.zip"
+    f.write_bytes(b"0123456789")
+    with _Server(_bucket()) as srv:
+        sent = s3.put_file_in_parts(srv.target(), "media/x.zip", f,
+                                    sha256=hashlib.sha256(b"0123456789").hexdigest())
+    assert sent == 10
+    assert [r["body"] for r in srv.seen if r["method"] == "PUT"] == [b"0123", b"4567", b"89"]
+    assert srv.seen[-1]["method"] == "POST"             # completed
+
+
+def test_a_file_that_changed_since_import_is_not_completed(tmp_path, monkeypatch):
+    monkeypatch.setattr(s3, "PART_SIZE", 4)
+    f = tmp_path / "big.zip"
+    f.write_bytes(b"0123456789")
+    with _Server(_bucket()) as srv:
+        with pytest.raises(s3.S3Error, match="recorded checksum"):
+            s3.put_file_in_parts(srv.target(), "media/x.zip", f, sha256="0" * 64)
+    assert srv.seen[-1]["method"] == "DELETE"           # aborted, nothing stored
+
+
+def test_a_paced_put_sends_the_same_bytes(tmp_path):
+    f = tmp_path / "a.bin"
+    f.write_bytes(b"z" * (s3.CHUNK + 5))
+    with _Server(lambda h: (200, b"")) as srv:
+        s3.put_file(srv.target(), "a.bin", f, pacer=s3.Pacer(0))
+    assert srv.seen[0]["body"] == b"z" * (s3.CHUNK + 5)
+
+
+def test_presence_is_the_exact_key_at_the_exact_size():
+    listing = (b"<ListBucketResult><Contents><Key>media/a.jpg</Key><Size>3</Size>"
+               b"</Contents></ListBucketResult>")
+    with _Server(lambda h: (200, listing)) as srv:
+        assert s3.object_present(srv.target(), "media/a.jpg", 3)
+        assert not s3.object_present(srv.target(), "media/a.jpg", 4)
+        assert not s3.object_present(srv.target(), "media/a.jp", 3)

tests/test_backup_service.py

@@ -5,6 +5,7 @@ without external binaries. The real subprocess behavior is exercised
 implicitly via the Celery task tests in test_tasks_backup.py.
 """
 import json
+import re
 import subprocess
 from pathlib import Path
 
@@ -28,7 +29,7 @@ def fake_subprocess(monkeypatch):
         stderr = b""
 
     def _write_sentinel(cmd):
-        if cmd[0] == "pg_dump":
+        if "pg_dump" in cmd:
             i = cmd.index("-f")
             Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
         elif cmd[0] == "tar" and "-cf" in cmd:
@@ -87,7 +88,7 @@ def test_backup_db_strips_sqlalchemy_psycopg_driver(tmp_path, fake_subprocess):
         db_url="postgresql+psycopg://u:p@h/d", images_root=tmp_path,
     )
     cmd = fake_subprocess[0]
-    assert cmd[0] == "pg_dump"
+    assert "pg_dump" in cmd
     assert cmd[-1].startswith("postgresql://")
     assert "+psycopg" not in cmd[-1]
 
@@ -284,3 +285,132 @@ def test_run_bounded_fails_fast_when_unkillable(monkeypatch):
     with pytest.raises(subprocess.TimeoutExpired):
         backup_service._run_bounded(["pg_dump"], 1)
     assert killed["n"] == 1
+
+
+# --- backup_db_offsite: pg_dump streamed to the bucket ---------------
+
+
+class _Dump:
+    """A pg_dump stand-in: `output` on stdout, `said` on stderr, exiting with
+    `code` once its output has been read."""
+
+    def __init__(self, output=b"", said=b"", code=0):
+        self.output, self.said, self.code = output, said, code
+        self.cmd = None
+        self.killed = False
+
+    def __call__(self, cmd, stdout=None, stderr=None, **kwargs):
+        import io
+        self.cmd = list(cmd)
+        self.args = cmd
+        self.stdout = io.BytesIO(self.output)
+        stderr.write(self.said)
+        self.returncode = None
+        return self
+
+    def poll(self):
+        return self.returncode
+
+    def wait(self, timeout=None):
+        if self.returncode is None:
+            self.returncode = -9 if self.killed else self.code
+        return self.returncode
+
+    def kill(self):
+        self.killed = True
+
+
+class _Bucket:
+    """Records what an Upload was asked to do instead of sending it."""
+
+    def __init__(self, fail_on_part=None):
+        self.fail_on_part = fail_on_part
+        self.uploads = []
+
+    def __call__(self, target, key, **kwargs):
+        from backend.app.services.backup_s3 import S3Error
+        bucket = self
+
+        class _Upload:
+            def __init__(self):
+                self.key, self.received = key, []
+                self.completed = self.aborted = False
+                bucket.uploads.append(self)
+
+            @property
+            def parts(self):
+                return len(self.received)
+
+            @property
+            def size(self):
+                return sum(len(p) for p in self.received)
+
+            def add(self, data):
+                if bucket.fail_on_part == len(self.received) + 1:
+                    raise S3Error("HTTP 503: busy.")
+                self.received.append(data)
+
+            def complete(self):
+                self.completed = True
+
+            def abort(self):
+                self.aborted = True
+
+        return _Upload()
+
+
+@pytest.fixture
+def offsite(monkeypatch):
+    from backend.app.services import backup_s3
+    monkeypatch.setattr(backup_s3, "PART_SIZE", 4)
+    bucket = _Bucket()
+    monkeypatch.setattr(backup_s3, "Upload", bucket)
+    target = backup_s3.S3Target("http://archives.example:8081", "curator", "AK", "SK")
+    return bucket, target
+
+
+def test_a_dump_streams_to_the_bucket_in_parts_with_nothing_kept_here(offsite, monkeypatch):
+    bucket, target = offsite
+    dump = _Dump(output=b"0123456789")
+    monkeypatch.setattr("subprocess.Popen", dump)
+    result = backup_service.backup_db_offsite(
+        db_url="postgresql+psycopg://u:p@h/d", target=target, limit_mbps=50,
+    )
+    (upload,) = bucket.uploads
+    assert upload.received == [b"0123", b"4567", b"89"]
+    assert upload.completed and not upload.aborted
+    assert re.fullmatch(r"database/fabledcurator-\d{8}T\d{6}Z\.dump", result["key"])
+    assert result["key"] == upload.key
+    assert (result["bucket"], result["size_bytes"], result["parts"]) == ("curator", 10, 3)
+    assert dump.cmd[0] == "nice" and "pg_dump" in dump.cmd and "-Fc" in dump.cmd
+    assert "-f" not in dump.cmd                  # stdout, not a file
+    assert dump.cmd[-1] == "postgresql://u:p@h/d"
+
+
+def test_a_failed_dump_leaves_nothing_in_the_bucket_and_says_why(offsite, monkeypatch):
+    bucket, target = offsite
+    said = b"pg_dump: dumping contents of table x\npg_dump: error: connection lost\n"
+    monkeypatch.setattr("subprocess.Popen", _Dump(output=b"012345", said=said, code=1))
+    with pytest.raises(backup_service.DumpFailed, match="^pg_dump: error: connection lost$"):
+        backup_service.backup_db_offsite(db_url="postgresql://u@h/d", target=target)
+    (upload,) = bucket.uploads
+    assert upload.aborted and not upload.completed
+
+
+def test_a_bucket_that_stops_taking_parts_stops_the_dump(offsite, monkeypatch):
+    from backend.app.services.backup_s3 import S3Error
+    bucket, target = offsite
+    bucket.fail_on_part = 2
+    dump = _Dump(output=b"0123456789")
+    monkeypatch.setattr("subprocess.Popen", dump)
+    with pytest.raises(S3Error):
+        backup_service.backup_db_offsite(db_url="postgresql://u@h/d", target=target)
+    assert dump.killed
+    (upload,) = bucket.uploads
+    assert upload.aborted and not upload.completed
+
+
+def test_dump_names_sort_by_time():
+    assert backup_service.offsite_dump_key("20261107T030000Z") == (
+        "database/fabledcurator-20261107T030000Z.dump"
+    )

tests/test_discord_client.py

@@ -153,9 +153,11 @@ def test_an_embeds_identity_ignores_its_signature():
     def embed(sig):
         return {"type": "image", "image": {"proxy_url": f"https://media/p/x.png?ex={sig}"}}
 
-    one = DiscordClient.extract_media(_msg(9, embeds=[embed("a")]))[0].media_id
-    two = DiscordClient.extract_media(_msg(9, embeds=[embed("b")]))[0].media_id
-    assert one == two and len(one) <= 33
+    art = [{"url": "https://cdn/a/1.png"}]
+    one = DiscordClient.extract_media(_msg(9, attachments=art, embeds=[embed("a")]))[1]
+    two = DiscordClient.extract_media(_msg(9, attachments=art, embeds=[embed("b")]))[1]
+    assert one.kind == two.kind == "embed"
+    assert one.media_id == two.media_id and len(one.media_id) <= 33
 
 
 def test_post_seams():
@@ -165,11 +167,52 @@ def test_post_seams():
 
 
 def test_a_text_only_message_is_not_a_post():
-    """gallery-dl never made one: chat lines would bury the drops."""
+    """Chat lines would bury the drops."""
     assert DiscordClient.post_record_key(_msg(6, content="brb")) is None
     assert DiscordClient.post_meta(_msg(1))["date"].startswith("2026-09-20")
 
 
+def test_a_message_without_an_attached_image_is_chat():
+    """Operator 2026-09-28: only content with an image attached. A lone
+    archive, a link preview or a Tenor GIF takes nothing and records nothing."""
+    chat = [
+        _msg(1, content="the stash", attachments=[
+            {"url": "https://cdn/a/Links_Stash.rar", "content_type": "application/x-rar"},
+        ]),
+        _msg(2, content="look", embeds=[
+            {"type": "image", "image": {"proxy_url": "https://media/p/x.png"}},
+        ]),
+        _msg(3, content="lol", embeds=[
+            {"type": "gifv", "video": {"proxy_url": "https://media/p/t.mp4"}},
+        ]),
+        _msg(4, content="wip", attachments=[{"url": "https://cdn/a/wip.psd"}]),
+    ]
+    for m in chat:
+        assert DiscordClient.extract_media(m) == []
+        assert DiscordClient.post_record_key(m) is None
+
+
+def test_an_attached_image_takes_every_file_numbered_as_gallery_dl_does():
+    """The gate decides WHETHER a message is taken, never which of its files:
+    the rar beside the image keeps its number, so on-disk names still match."""
+    m = _msg(7, attachments=[
+        {"url": "https://cdn/a/pack.rar"},
+        {"url": "https://cdn/a/noext", "content_type": "image/png"},
+    ])
+    items = DiscordClient.extract_media(m)
+    assert [(i.num, i.filename) for i in items] == [(1, "pack"), (2, "noext")]
+    assert DiscordClient.post_record_key(m) == ("message:7", "7")
+    video = _msg(8, attachments=[{"url": "https://cdn/a/clip.MP4?ex=1"}])
+    assert len(DiscordClient.extract_media(video)) == 1
+
+
+def test_a_forwarded_image_counts_as_attached():
+    m = _msg(9, message_snapshots=[
+        {"message": {"type": 0, "attachments": [{"url": "https://cdn/a/fwd.png"}]}},
+    ])
+    assert DiscordClient.post_record_key(m) == ("message:9", "9")
+
+
 # -- the walk --------------------------------------------------------------------
 
 def test_a_channel_pages_newest_first_and_skips_system_messages():
@@ -206,6 +249,50 @@ def test_messages_carry_server_and_channel_metadata():
     assert meta["parent"] == "Art"
 
 
+def _one_channel(messages):
+    return {
+        "/guilds/1": _ok({"id": "1", "name": "Todding's Server"}),
+        "/guilds/1/channels": _ok([
+            {"id": "4", "type": 4, "name": "Todding"},
+            {"id": "2", "type": 0, "name": "banana-land", "parent_id": "4"},
+        ]),
+        ("/channels/2/messages", None): _ok(messages),
+        ("/channels/2/threads/search", 0): _ok({"threads": []}),
+    }
+
+
+def test_only_from_takes_the_creators_messages_and_no_one_elses():
+    """Operator 2026-09-28: in the creator's server, another member posting a
+    meme is not the creator's art. Matched by id, username or display name."""
+    creator = {"id": "7", "username": "todding", "global_name": "Todding"}
+    other = {"id": "8", "username": "jakeboii", "global_name": "Jake Boii"}
+    msgs = [_msg(3, author=creator), _msg(2, author=other), _msg(1, author=creator)]
+    url = "https://discord.com/channels/1/2"
+
+    everyone = _client(_one_channel([dict(m) for m in msgs]))
+    assert [mid for mid, _ in _ids(everyone, url)] == ["3", "2", "1"]
+    for who in (["Todding"], ["TODDING "], ["7"]):
+        client = _client(_one_channel([dict(m) for m in msgs]))
+        client.only_from(who)
+        assert [mid for mid, _ in _ids(client, url)] == ["3", "1"]
+    client = _client(_one_channel([dict(m) for m in msgs]))
+    client.only_from([])
+    assert len(_ids(client, url)) == 3
+
+
+def test_the_source_label_names_the_server_and_channel_the_walk_read():
+    client = _client(_one_channel([]))
+    url = "https://discord.com/channels/1/2"
+    assert client.source_label(url) is None  # nothing walked yet
+    list(client.iter_posts(url))
+    assert client.source_label(url) == "Todding's Server · #banana-land"
+    assert client.source_label("https://discord.com/channels/1") == "Todding's Server"
+    client._channels["9"] = {"channel": "wip", "is_thread": True, "parent": "banana-land"}
+    assert client.source_label("https://discord.com/channels/1/9") == (
+        "Todding's Server · #banana-land › wip"
+    )
+
+
 def test_a_server_walks_text_then_threads_newest_created_first_and_skips_private():
     routes = {
         "/guilds/1": _ok({"id": "1", "name": "S"}),
@@ -337,3 +424,60 @@ def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path)
         _msg(9, attachments=[{"id": "300", "url": "https://cdn/a/1.png"}])
     )
     assert _ledger_key(media) == "9:300"
+
+
+# -- the poster picker (#4491) -------------------------------------------------
+
+def _person(pid, username, global_name=None, **extra):
+    return {"id": pid, "username": username, "global_name": global_name,
+            "messages": 1, "images": 0, **extra}
+
+
+def test_the_creator_a_server_is_named_for_is_suggested():
+    ranked = dc.rank_posters("Todding's Server", "99", [
+        _person("8", "jakeboii", "Jake Boii", images=3),
+        _person("7", "todding", "Todding", images=1),
+    ])
+    assert ranked[0]["id"] == "7" and ranked[0]["suggested"] is True
+    assert ranked[0]["reasons"] == ["name matches the server"]
+    assert ranked[1]["suggested"] is False
+
+
+def test_owning_the_server_and_matching_its_name_outranks_either_alone():
+    ranked = dc.rank_posters("The Official Todding Discord", "7", [
+        _person("5", "toddingfan"),
+        _person("7", "t0dd", "Todding"),
+    ])
+    assert [r["id"] for r in ranked if r["suggested"]] == ["7"]
+    assert ranked[0]["reasons"] == ["owns the server", "name matches the server"]
+
+
+def test_no_signal_suggests_no_one_and_bots_never():
+    assert not any(r["suggested"] for r in dc.rank_posters("Art Club", None, [
+        _person("1", "alice"), _person("2", "bob"),
+    ]))
+    ranked = dc.rank_posters("Todding's Server", None, [_person("3", "toddingbot", bot=True)])
+    assert ranked[0]["suggested"] is False
+
+
+def test_recent_posters_tallies_a_shallow_window_by_id():
+    creator = {"id": "7", "username": "todding", "global_name": "Todding"}
+    other = {"id": "8", "username": "jakeboii", "global_name": "Jake Boii"}
+    art = [{"url": "https://cdn/a/1.png"}]
+    page = [_msg(i, author=creator, attachments=art) for i in range(300, 240, -1)]
+    page += [_msg(i, author=other) for i in range(240, 200, -1)]
+    routes = {
+        "/guilds/1": _ok({"id": "1", "name": "Todding's Server", "owner_id": "7"}),
+        ("/channels/2/messages", None): _ok(page),
+        ("/channels/2/messages", "201"): _ok([_msg(150, author=other)]),
+    }
+    client = _client(routes)
+    got = client.recent_posters("1", "2", max_messages=150)
+    assert got["server"] == "Todding's Server" and got["owner_id"] == "7"
+    assert got["scanned"] == 101
+    first = got["posters"][0]
+    assert (first["id"], first["messages"], first["images"], first["suggested"]) == ("7", 60, 60, True)
+    assert got["posters"][1]["messages"] == 41
+    # It asked for no more than the window: 100, then the 50 left of 150.
+    limits = [p.get("limit") for e, p in client._session.calls if e.endswith("/messages")]
+    assert limits == [100, 50]

tests/test_discord_poster_cleanup.py

@@ -0,0 +1,131 @@
+"""Removing a Discord source's posts by people outside its poster list (#4486).
+
+Real Postgres (db_sync): the predicates read `raw_metadata->>'author'`, which
+only the database can evaluate. The test that matters runs preview and apply
+on one fixture and asserts they agree AND that the rows went (snippet #3087).
+"""
+
+import pytest
+from sqlalchemy import func, select
+
+from backend.app.models import (
+    Artist,
+    ImageProvenance,
+    ImageRecord,
+    Post,
+    PostAttachment,
+    Source,
+)
+from backend.app.services import discord_poster_cleanup as cleanup
+
+pytestmark = pytest.mark.integration
+
+
+def _image(db, artist, tmp_path, name, post):
+    f = tmp_path / f"{name}.png"
+    f.write_bytes(b"x")
+    img = ImageRecord(
+        artist_id=artist.id, path=str(f), sha256=name.ljust(64, "0"),
+        size_bytes=1, mime="image/png", origin="downloaded", primary_post_id=post.id,
+    )
+    db.add(img)
+    db.flush()
+    return img
+
+
+def _post(db, source, mid, raw, **extra):
+    p = Post(
+        source_id=source.id, artist_id=source.artist_id, external_post_id=mid,
+        raw_metadata=raw, **extra,
+    )
+    db.add(p)
+    db.flush()
+    return p
+
+
+def _fixture(db, tmp_path, authors=("Todding",)):
+    """Todding's post, two of Jake's (one sharing an image with Todding's), an
+    old post with no poster, and a drop that absorbed Todding's and Jake's."""
+    artist = Artist(name="Todding", slug="todding-cleanup")
+    db.add(artist)
+    db.flush()
+    src = Source(
+        artist_id=artist.id, platform="discord",
+        url="https://discord.com/channels/1/2",
+        config_overrides={"discord_authors": list(authors)} if authors else None,
+    )
+    db.add(src)
+    db.flush()
+    todd = _post(db, src, "10", {"author": "todding", "author_name": "Todding", "author_id": "1"})
+    jake1 = _post(db, src, "11", {"author": "jakeboii", "author_id": "8"})
+    jake2 = _post(db, src, "12", {"author": "jakeboii", "author_id": "8"})
+    old = _post(db, src, "13", {"category": "discord"})
+    drop = _post(db, src, "fc-drop:10", None, synthesized_by="discord_drop")
+    todd.absorbed_by_post_id = drop.id
+    jake1.absorbed_by_post_id = drop.id
+
+    art = _image(db, artist, tmp_path, "art", todd)
+    meme = _image(db, artist, tmp_path, "meme", jake1)
+    shared = _image(db, artist, tmp_path, "shared", jake2)
+    legacy = _image(db, artist, tmp_path, "legacy", old)
+    db.add_all([
+        ImageProvenance(image_record_id=shared.id, post_id=todd.id, source_id=src.id),
+        ImageProvenance(image_record_id=art.id, post_id=drop.id, source_id=src.id),
+        ImageProvenance(image_record_id=meme.id, post_id=drop.id, source_id=src.id),
+        PostAttachment(
+            post_id=jake2.id, artist_id=artist.id, sha256="z" * 64,
+            path="/store/z/f.zip", original_filename="f.zip", ext=".zip", size_bytes=1,
+        ),
+    ])
+    db.commit()
+    return src, {
+        "todd": todd.id, "jake1": jake1.id, "jake2": jake2.id, "old": old.id,
+        "drop": drop.id, "art": art.id, "meme": meme.id, "shared": shared.id,
+        "legacy": legacy.id,
+    }
+
+
+def test_preview_matches_apply_and_the_rows_go(db_sync, tmp_path):
+    src, ids = _fixture(db_sync, tmp_path)
+
+    projected = cleanup.preview(db_sync, source_id=src.id)
+    assert projected["authors"] == ["todding"]
+    assert projected["posters"] == [{"poster": "jakeboii", "posts": 2}]
+    assert projected["unknown_posts"] == 1
+
+    done = cleanup.apply(db_sync, source_id=src.id, images_root=tmp_path)
+    assert projected["posts"] == done["posts_deleted"] == 2
+    assert projected["images"] == done["images_deleted"] == 1  # the meme only
+    assert projected["attachments"] == done["attachments_deleted"] == 1
+    assert projected["drops"] == done["drops_deleted"] == 1
+
+    post_ids = set(db_sync.execute(select(Post.id).where(Post.source_id == src.id)).scalars())
+    assert post_ids == {ids["todd"], ids["old"]}
+    image_ids = set(db_sync.execute(
+        select(ImageRecord.id).where(ImageRecord.id.in_(
+            [ids["art"], ids["meme"], ids["shared"], ids["legacy"]]))
+    ).scalars())
+    # The meme was only Jake's. The shared piece is also on Todding's post, and
+    # the drop's link to the art was never what kept it.
+    assert image_ids == {ids["art"], ids["shared"], ids["legacy"]}
+    assert not (tmp_path / "meme.png").exists()
+    # Todding's message is back in the feed, for the sweep to regroup.
+    assert db_sync.get(Post, ids["todd"]).absorbed_by_post_id is None
+    assert db_sync.execute(select(func.count(PostAttachment.id)).where(
+        PostAttachment.sha256 == "z" * 64)).scalar_one() == 0
+
+
+def test_a_poster_matches_by_display_name_or_id_too(db_sync, tmp_path):
+    src, _ = _fixture(db_sync, tmp_path, authors=("8",))
+    projected = cleanup.preview(db_sync, source_id=src.id)
+    assert projected["posters"] == [{"poster": "Todding", "posts": 1}]
+
+
+def test_no_list_means_nothing_to_remove(db_sync, tmp_path):
+    src, _ = _fixture(db_sync, tmp_path, authors=())
+    with pytest.raises(cleanup.PosterCleanupError):
+        cleanup.preview(db_sync, source_id=src.id)
+
+
+def test_the_token_names_what_the_preview_showed():
+    assert cleanup.confirm_token({"posts": 2, "images": 1}) == "remove-2-posts-1-images"

tests/test_download_service.py

@@ -193,6 +193,69 @@ async def test_download_source_attaches_written_files(
     assert imported_when_marked == [2]
 
 
+@pytest.mark.asyncio
+async def test_import_step_keeps_only_the_files_the_library_needs_on_disk(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """After the import step, a downloaded image the library took stays where it
+    is, while an archive whose images were extracted (the archive is in the
+    attachments store) and an image the import filters rejected are removed, so
+    nothing is left on disk without a record for a later walk to import again."""
+    import zipfile
+
+    from PIL import Image
+
+    from backend.app.models import PostAttachment
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    images_root = tmp_path / "images"
+    post_dir = images_root / "alice" / "patreon" / "post"
+
+    kept = post_dir / "a.jpg"
+    _make_jpg(kept, split="h")
+    member = tmp_path / "member.jpg"
+    _make_jpg(member, split="v")
+    archive = post_dir / "pack.zip"
+    with zipfile.ZipFile(archive, "w") as z:
+        z.writestr("art.jpg", member.read_bytes())
+    tiny = post_dir / "tiny.png"
+    Image.new("RGB", (16, 16), (200, 30, 30)).save(tiny)  # below min_width
+
+    result = _make_fake_dl_result(
+        success=True, written_paths=[str(kept), str(archive), str(tiny)],
+        files_downloaded=3,
+    )
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    sync_settings.phash_threshold = 0
+    sync_settings.min_width = 32
+    importer = Importer(
+        session=db_sync, images_root=images_root, import_root=images_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
+    )
+    crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync, gdl=_fake_gdl_with_result(result),
+        importer=importer, cred_service=CredentialService(db, crypto),
+    )
+    _stub_patreon_ingester(svc, result)
+    await svc.download_source(source.id)
+
+    assert kept.exists()
+    assert not archive.exists()
+    assert not tiny.exists()
+    # The archive and its image are both still in the library.
+    assert db_sync.execute(
+        select(func.count(PostAttachment.id)).where(
+            PostAttachment.original_filename == "pack.zip"
+        )
+    ).scalar_one() == 1
+    assert db_sync.execute(select(func.count(ImageRecord.id))).scalar_one() == 2
+
+
 @pytest.mark.asyncio
 async def test_download_source_skipped_when_disabled(
     db, db_sync, tmp_path, seed_artist_and_source,
@@ -570,6 +633,64 @@ async def test_backfill_chunk_progress_advances_cursor(
     assert remaining == 4
 
 
+@pytest.mark.asyncio
+async def test_phase_3_leaves_imports_past_its_deadline_for_the_next_chunk(
+    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
+):
+    """A walk that reached its end handed phase 3 more imports than fit before
+    the deadline. Phase 3 imports what fits, leaves the rest on disk and
+    unmarked, keeps the backfill running, and resumes it on the page holding
+    the first file it left."""
+    import backend.app.services.download_service as ds
+
+    _artist, source = seed_artist_and_source
+    source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:PAGE1"}
+    source.backfill_runs_remaining = 5
+    await db.commit()
+
+    images_root = tmp_path / "images"
+    f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
+    f2 = images_root / "alice" / "patreon" / "post" / "b.jpg"
+    _make_jpg(f1, split="h")
+    _make_jpg(f2, split="v")
+    result = _make_fake_dl_result(
+        success=True, written_paths=[str(f1), str(f2)], cursor="03:PAGE3",
+    )
+    result.written_cursors = ["03:PAGE1", "03:PAGE2"]
+    left_unmarked = []
+    result.mark_seen_after_import = lambda skip_paths=(): left_unmarked.append(
+        set(skip_paths)
+    )
+
+    # The clock is past the deadline once the first import is done.
+    past = ds.PHASE3_DEADLINE_SECONDS + 1
+    ticks = iter([0.0, 0.0])
+    monkeypatch.setattr(ds, "time", SimpleNamespace(monotonic=lambda: next(ticks, past)))
+
+    svc, _ = _backfill_svc(db, db_sync, tmp_path, result)
+    event_id = await svc.download_source(source.id)
+
+    ev = (await db.execute(
+        select(DownloadEvent).where(DownloadEvent.id == event_id)
+    )).scalar_one()
+    assert ev.status == "ok"
+    assert ev.files_count == 1
+    assert "1 file(s) left for the next chunk" in ev.metadata_["stdout"]
+    assert left_unmarked == [{str(f2)}]
+    assert f2.exists()
+    assert db_sync.execute(
+        select(func.count(ImageRecord.id)).where(ImageRecord.path == str(f2))
+    ).scalar_one() == 0
+
+    remaining, co = (await db.execute(
+        select(Source.backfill_runs_remaining, Source.config_overrides)
+        .where(Source.id == source.id)
+    )).one()
+    assert co.get("_backfill_state") == "running"
+    assert co.get("_backfill_cursor") == "03:PAGE2"
+    assert remaining == 4
+
+
 @pytest.mark.asyncio
 async def test_backfill_state_running_selects_backfill_mode_and_resumes(
     db, db_sync, tmp_path, seed_artist_and_source,

tests/test_download_source_task.py

@@ -55,6 +55,15 @@ def test_the_native_chunk_leaves_room_to_tear_down_before_the_soft_limit():
     assert CHUNK_TOTAL_SECONDS <= DOWNLOAD_SOFT_TIME_LIMIT - 120
 
 
+def test_phase_3_stops_importing_with_room_to_finalize():
+    """Past its deadline phase 3 starts no import; what remains after it (the
+    import in hand, post records, the event) has to fit before the soft limit."""
+    from backend.app.services.ingest_core import PHASE3_DEADLINE_SECONDS
+    from backend.app.tasks.download import DOWNLOAD_SOFT_TIME_LIMIT
+
+    assert PHASE3_DEADLINE_SECONDS <= DOWNLOAD_SOFT_TIME_LIMIT - 120
+
+
 def test_decorated_limits_match_module_constants():
     """The @celery.task decorator must use the audited constants, not
     drifted literals."""
@@ -158,7 +167,7 @@ async def test_finalize_soft_limited_is_noop_without_running_event(db_sync):
 @pytest.mark.integration
 @pytest.mark.asyncio
 async def test_download_source_catches_soft_limit_and_salvages_event(
-    db_sync, monkeypatch,
+    db_sync, monkeypatch, celery_conf,
 ):
     """End-to-end wiring: when the inner run raises SoftTimeLimitExceeded,
     download_source's handler must flip the in-flight event to error
@@ -169,8 +178,7 @@ async def test_download_source_catches_soft_limit_and_salvages_event(
     import backend.app.tasks.download as dl
     from backend.app.models import DownloadEvent
 
-    monkeypatch.setattr(celery.conf, "task_always_eager", True)
-    monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
+    celery_conf(task_always_eager=True, task_eager_propagates=False)
     # This test exercises the soft-limit salvage path, not the per-platform
     # serialization. Neutralize the Redis lock so it can't defer/recurse under
     # eager mode (and so it doesn't contend with test_platform_lock).

tests/test_external_fetch.py

@@ -108,6 +108,27 @@ def test_mega_collects_new_files(tmp_path, monkeypatch):
     assert res.files[0].name == "film.mp4"
 
 
+def test_mega_folder_link_collects_files_in_subfolders(tmp_path, monkeypatch):
+    """A folder link keeps the folder's tree; every file in it is collected,
+    files already there before the fetch and staging files are not."""
+    (tmp_path / "old.png").write_bytes(b"earlier")
+
+    def fake_mega(url, out_dir, *, timeout):
+        nested = Path(out_dir) / "images" / "2020"
+        nested.mkdir(parents=True)
+        (nested / "kara.png").write_bytes(b"png")
+        (Path(out_dir) / "videos").mkdir()
+        (Path(out_dir) / "videos" / "loop.webm").write_bytes(b"webm")
+        (nested / ".megatmp.AbCd").write_bytes(b"half")
+
+    monkeypatch.setattr(ef, "_run_mega_get", fake_mega)
+    res = fetch_external("mega", "https://mega.nz/folder/x#k", tmp_path)
+    assert res.ok
+    assert sorted(p.relative_to(tmp_path).as_posix() for p in res.files) == [
+        "images/2020/kara.png", "videos/loop.webm",
+    ]
+
+
 def test_unsupported_host(tmp_path):
     res = fetch_external("nope", "https://x", tmp_path)
     assert not res.ok

tests/test_external_worker.py

@@ -251,3 +251,293 @@ def test_downloaded_archive_gets_provenance_and_tagging(db_sync, tmp_path, monke
     # Tagging + thumbnails queued for exactly those member images.
     assert set(tagged) == member_ids
     assert set(thumbed) == member_ids
+
+
+def _link_dir(tmp_path, host, link_id):
+    return tmp_path / f"ext-{host}" / "patreon" / "external" / f"EXT-{host}" / str(link_id)
+
+
+def test_extracted_archive_is_not_left_in_the_link_dir(db_sync, tmp_path, monkeypatch):
+    """An archive whose images were extracted is in the attachments store, so
+    the downloaded copy is removed instead of sitting beside it."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    written = []
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        zpath = dest_dir / "pack.zip"
+        with zipfile.ZipFile(zpath, "w") as z:
+            z.writestr("art.jpg", _structured_jpeg_bytes())
+        written.append(zpath)
+        return FetchResult(files=[zpath], bytes=zpath.stat().st_size)
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    import backend.app.tasks.ml as ml_mod
+    import backend.app.tasks.thumbnail as thumb_mod
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
+    monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
+
+    out = ext.fetch_external_link(link_id)
+    assert out["images"] >= 1
+    assert not written[0].exists()
+    db_sync.expire_all()
+    assert db_sync.execute(
+        select(func.count(PostAttachment.id))
+        .where(PostAttachment.original_filename == "pack.zip")
+    ).scalar_one() == 1
+
+
+def test_a_failed_fetch_leaves_no_partial_files(db_sync, tmp_path, monkeypatch):
+    """A fetch that fails midway (megatools leaves `.megatmp.*` behind) clears
+    the link's staging dir, so a dead link leaves nothing on disk."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        (dest_dir / ".megatmp.AbCd").write_bytes(b"half a game build")
+        return FetchResult(error="megatools dl exit 1: timed out")
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    ext.fetch_external_link(link_id)
+
+    link_dir = _link_dir(tmp_path, "mega", link_id)
+    assert not (link_dir / ".megatmp.AbCd").exists()
+    db_sync.expire_all()
+    assert db_sync.get(ExternalLink, link_id).status == "failed"
+
+
+def test_a_retry_clears_what_a_killed_attempt_left(db_sync, tmp_path, monkeypatch):
+    """Leftovers from an attempt whose worker was killed are removed before the
+    next fetch starts; an image imported in place by an earlier attempt stays."""
+    _, link = _seed(db_sync, host="pixeldrain")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    link_dir = _link_dir(tmp_path, "pixeldrain", link_id)
+    link_dir.mkdir(parents=True)
+    leftover = link_dir / "build.zip.part"
+    leftover.write_bytes(b"partial")
+    imported = link_dir / "kept.jpg"
+    imported.write_bytes(_structured_jpeg_bytes())
+    db_sync.add(ImageRecord(
+        path=str(imported), sha256="ext-kept".ljust(64, "x"),
+        size_bytes=imported.stat().st_size, mime="image/jpeg",
+        width=256, height=256, origin="downloaded", integrity_status="ok",
+    ))
+    db_sync.commit()
+    seen_at_fetch = []
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        seen_at_fetch.extend(sorted(p.name for p in dest_dir.iterdir()))
+        return FetchResult(error="HTTP 503")
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    ext.fetch_external_link(link_id)
+
+    assert seen_at_fetch == ["kept.jpg"]
+    assert imported.exists()
+    assert not leftover.exists()
+
+
+def test_a_rejected_download_is_removed(db_sync, tmp_path, monkeypatch):
+    """A downloaded image the import filters reject has no record, so it is
+    removed rather than left in the link's dir."""
+    from backend.app.models import ImportSettings
+
+    _, link = _seed(db_sync, host="dropbox")
+    link_id = link.id
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    settings.min_width = 512
+    db_sync.commit()
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    written = []
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        f = dest_dir / "small.jpg"  # 256px wide, below min_width
+        f.write_bytes(_structured_jpeg_bytes())
+        written.append(f)
+        return FetchResult(files=[f], bytes=f.stat().st_size)
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    out = ext.fetch_external_link(link_id)
+
+    assert out["images"] == 0
+    assert not written[0].exists()
+    db_sync.expire_all()
+    assert db_sync.get(ExternalLink, link_id).status == "downloaded"
+
+
+def _other_jpeg_bytes():
+    # A second structured image, unlike _structured_jpeg_bytes, so the two
+    # don't near-duplicate each other at import.
+    im = Image.new("RGB", (256, 256))
+    px = im.load()
+    for y in range(256):
+        for x in range(256):
+            px[x, y] = ((x * x) % 256, (y * 3) % 256, (x ^ y) % 256)
+    buf = io.BytesIO()
+    im.save(buf, "JPEG")
+    return buf.getvalue()
+
+
+def test_a_failed_fetch_keeps_complete_files(db_sync, tmp_path, monkeypatch):
+    """A failed fetch removes staging files only. A complete file may be the
+    only copy left once the host has dropped the upload."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        (dest_dir / "images").mkdir(parents=True, exist_ok=True)
+        (dest_dir / "images" / "done.jpg").write_bytes(_structured_jpeg_bytes())
+        (dest_dir / "images" / ".megatmp.AbCd").write_bytes(b"half")
+        return FetchResult(error="API call 'f' failed: ENOENT")
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    ext.fetch_external_link(link_id)
+
+    link_dir = _link_dir(tmp_path, "mega", link_id)
+    assert (link_dir / "images" / "done.jpg").exists()
+    assert not (link_dir / "images" / ".megatmp.AbCd").exists()
+
+
+def test_a_successful_fetch_imports_what_an_earlier_attempt_finished(
+    db_sync, tmp_path, monkeypatch,
+):
+    """Complete files an earlier attempt wrote, in subfolders too, are imported
+    with the new fetch's files instead of being deleted or left behind."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    link_dir = _link_dir(tmp_path, "mega", link_id)
+    (link_dir / "pack").mkdir(parents=True)
+    earlier = link_dir / "pack" / "earlier.jpg"
+    earlier.write_bytes(_structured_jpeg_bytes())
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        f = dest_dir / "new.jpg"
+        f.write_bytes(_other_jpeg_bytes())
+        return FetchResult(files=[f], bytes=f.stat().st_size)
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    import backend.app.tasks.ml as ml_mod
+    import backend.app.tasks.thumbnail as thumb_mod
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
+    monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
+
+    out = ext.fetch_external_link(link_id)
+
+    assert out["images"] == 2
+    db_sync.expire_all()
+    paths = set(db_sync.execute(select(ImageRecord.path)).scalars())
+    assert {str(earlier), str(link_dir / "new.jpg")} <= paths
+
+
+def test_sweep_recovers_a_link_whose_worker_died(db_sync, monkeypatch):
+    """A link 'downloading' past the task's hard limit (or with no claim time)
+    counts as a failed attempt and is retried; a fresh claim is left alone."""
+    from datetime import UTC, datetime, timedelta
+
+    _, stale = _seed(db_sync, host="mega", status="downloading")
+    _, unclaimed = _seed(db_sync, host="gdrive", status="downloading")
+    _, fresh = _seed(db_sync, host="dropbox", status="downloading")
+    now = datetime.now(UTC)
+    stale.started_at = now - timedelta(minutes=ext._STALL_MINUTES + 5)
+    fresh.started_at = now - timedelta(minutes=5)
+    db_sync.commit()
+    stale_id, unclaimed_id, fresh_id = stale.id, unclaimed.id, fresh.id
+
+    enqueued = []
+    monkeypatch.setattr(ext.fetch_external_link, "delay", lambda lid: enqueued.append(lid))
+    out = ext.sweep_external_links()
+
+    assert out["recovered"] == 2
+    assert set(enqueued) == {stale_id, unclaimed_id}
+    db_sync.expire_all()
+    for lid in (stale_id, unclaimed_id):
+        row = db_sync.get(ExternalLink, lid)
+        assert (row.status, row.attempts) == ("failed", 1)
+    assert db_sync.get(ExternalLink, fresh_id).status == "downloading"
+
+
+def test_recovery_gives_a_dead_link_one_more_attempt_at_growing_intervals(db_sync, monkeypatch):
+    """A dead link is retried once after 2**recoveries days, not given a fresh
+    three-attempt budget, so a link that always fails isn't downloaded daily."""
+    from datetime import UTC, datetime, timedelta
+
+    now = datetime.now(UTC)
+    _, due = _seed(db_sync, host="mega", status="dead")
+    _, waiting = _seed(db_sync, host="gdrive", status="dead")
+    _, spent = _seed(db_sync, host="dropbox", status="dead")
+    due.attempts = waiting.attempts = spent.attempts = ext.DEAD_LETTER_THRESHOLD
+    due.recoveries, due.completed_at = 2, now - timedelta(days=4, hours=1)
+    waiting.recoveries, waiting.completed_at = 2, now - timedelta(days=3)
+    spent.recoveries, spent.completed_at = ext._MAX_RECOVERIES, now - timedelta(days=90)
+    db_sync.commit()
+    due_id, waiting_id, spent_id = due.id, waiting.id, spent.id
+
+    enqueued = []
+    monkeypatch.setattr(ext.fetch_external_link, "delay", lambda lid: enqueued.append(lid))
+    out = ext.recover_external_links()
+
+    assert out["revived"] == 1
+    assert enqueued == [due_id]
+    db_sync.expire_all()
+    row = db_sync.get(ExternalLink, due_id)
+    assert (row.status, row.attempts, row.recoveries) == (
+        "failed", ext.DEAD_LETTER_THRESHOLD - 1, 3,
+    )
+    assert db_sync.get(ExternalLink, waiting_id).status == "dead"
+    assert db_sync.get(ExternalLink, spent_id).status == "dead"
+
+
+def test_a_recovered_link_that_fails_again_goes_straight_back_to_dead(
+    db_sync, tmp_path, monkeypatch,
+):
+    _, link = _seed(db_sync, host="mega", status="dead")
+    link.attempts = ext.DEAD_LETTER_THRESHOLD
+    db_sync.commit()
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    monkeypatch.setattr(ext, "fetch_external", lambda *a, **k: FetchResult(error="too big"))
+    monkeypatch.setattr(ext.fetch_external_link, "delay", ext.fetch_external_link)
+
+    ext.recover_external_links()
+
+    db_sync.expire_all()
+    row = db_sync.get(ExternalLink, link_id)
+    assert (row.status, row.recoveries) == ("dead", 1)
+
+
+def test_prune_removes_only_links_recovery_has_given_up_on(db_sync):
+    from datetime import UTC, datetime, timedelta
+
+    old = datetime.now(UTC) - timedelta(days=ext._RETENTION_DAYS + 1)
+    _, spent = _seed(db_sync, host="mega", status="dead")
+    _, recovering = _seed(db_sync, host="gdrive", status="dead")
+    _, recent = _seed(db_sync, host="dropbox", status="dead")
+    spent.recoveries, spent.completed_at = ext._MAX_RECOVERIES, old
+    recovering.recoveries, recovering.completed_at = 1, old
+    recent.recoveries = ext._MAX_RECOVERIES
+    recent.completed_at = datetime.now(UTC) - timedelta(days=1)
+    db_sync.commit()
+    spent_id, recovering_id, recent_id = spent.id, recovering.id, recent.id
+
+    assert ext.prune_external_links() == {"pruned": 1}
+    db_sync.expire_all()
+    assert db_sync.get(ExternalLink, spent_id) is None
+    assert db_sync.get(ExternalLink, recovering_id) is not None
+    assert db_sync.get(ExternalLink, recent_id) is not None

tests/test_importer_attach_in_place.py

@@ -410,6 +410,31 @@ def test_attach_in_place_invalid_image_returns_skipped(importer):
     assert result.skip_reason.value == "invalid_image"
 
 
+@pytest.mark.parametrize("name", ["film.mp4abc123_.part", "img.jpg.partial", ".megatmp.Xy12"])
+def test_an_unfinished_download_is_refused_and_left_on_disk(importer, db_sync, name):
+    """A staging file is a download still being written (or abandoned): it is
+    never stored as an attachment, and the importer never deletes it, so a
+    download in progress is not pulled out from under its writer."""
+    from backend.app.models import PostAttachment
+
+    artist = Artist(name="Hal", slug="hal")
+    db_sync.add(artist)
+    db_sync.flush()
+    staging = importer.images_root / "hal" / "patreon" / "post" / name
+    staging.parent.mkdir(parents=True, exist_ok=True)
+    staging.write_bytes(b"half of a video")
+
+    for result in (importer.attach_in_place(staging, artist=artist),
+                   importer.import_one(staging)):
+        assert result.status == "skipped"
+        assert "unfinished download" in result.error
+        assert importer.discard_after_attach(staging, result) is False
+    assert staging.exists()
+    assert db_sync.execute(
+        select(func.count()).where(PostAttachment.original_filename == name)
+    ).scalar() == 0
+
+
 def test_attach_in_place_non_media_routes_to_attachment(importer, db_sync):
     """FC-2d-iii dispatch parity for the download path. A non-media file
     (.pdf, .txt etc.) downloaded by gallery-dl must become a PostAttachment,

tests/test_importer_sha_race.py

@@ -0,0 +1,202 @@
+"""Two workers importing the same picture at once (services/importer.py).
+
+The importer looks a file's sha256 up before inserting its row. When two copies
+of one picture are imported side by side, both lookups can find nothing; the
+unique index on sha256 then refuses the second insert. The loser files its copy
+as a duplicate of the winner, links its post, and leaves the session usable,
+instead of failing the import with a dead transaction.
+
+The race is staged the way tests/test_importer_provenance_race.py stages its
+own: the loser's sha256 lookup is forced to miss once, as it would if it ran
+just before the winner's commit. The loser's near-duplicate cache is empty for
+the same reason (it was loaded before the winner's row existed).
+"""
+
+import io
+import zipfile
+from pathlib import Path
+
+import pytest
+from PIL import Image
+from sqlalchemy import func, select
+
+from backend.app.models import Artist, ImageRecord, ImportSettings, PostAttachment
+from backend.app.services.importer import Importer, SkipReason
+from backend.app.services.thumbnailer import Thumbnailer
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def layout(tmp_path):
+    import_root = tmp_path / "import"
+    images_root = tmp_path / "images"
+    import_root.mkdir()
+    images_root.mkdir()
+    return import_root, images_root
+
+
+def _worker(db_sync, layout) -> Importer:
+    import_root, images_root = layout
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    return Importer(
+        session=db_sync, images_root=images_root, import_root=import_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
+    )
+
+
+def _jpeg(path: Path) -> bytes:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    Image.new("RGB", (320, 240), (40, 90, 160)).save(path, "JPEG")
+    return path.read_bytes()
+
+
+def _copy(src: Path, dst: Path) -> Path:
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    dst.write_bytes(src.read_bytes())
+    return dst
+
+
+def _lose_the_race(importer: Importer, monkeypatch) -> None:
+    """Make `importer` miss its first sha256 lookup and see no near-duplicates."""
+    real_execute = importer.session.execute
+    missed = [False]
+
+    class _Miss:
+        def scalar_one_or_none(self):
+            return None
+
+    def execute(stmt, *args, **kwargs):
+        sql = str(stmt)
+        if not missed[0] and "FROM image_record" in sql and "image_record.sha256 =" in sql:
+            missed[0] = True
+            return _Miss()
+        return real_execute(stmt, *args, **kwargs)
+
+    monkeypatch.setattr(importer.session, "execute", execute)
+    monkeypatch.setattr(importer, "_phash_candidates_cache", lambda: [])
+
+
+def _records(db_sync) -> int:
+    return db_sync.execute(select(func.count()).select_from(ImageRecord)).scalar_one()
+
+
+def test_a_copy_imported_alongside_the_original_is_filed_as_its_duplicate(
+    db_sync, layout, monkeypatch,
+):
+    import_root, images_root = layout
+    first = import_root / "Alice" / "set1" / "pic.jpg"
+    _jpeg(first)
+    second = _copy(first, import_root / "Alice" / "set2" / "pic.jpg")
+
+    winner = _worker(db_sync, layout).import_one(first)
+    assert winner.status == "imported"
+    kept = Path(db_sync.get(ImageRecord, winner.image_id).path)
+
+    loser = _worker(db_sync, layout)
+    _lose_the_race(loser, monkeypatch)
+    result = loser.import_one(second)
+
+    assert result.status == "skipped"
+    assert result.skip_reason == SkipReason.duplicate_hash
+    assert result.image_id == winner.image_id
+    monkeypatch.undo()
+    assert _records(db_sync) == 1
+    assert kept.is_file()
+    # The loser's copy in the library was a stray and is gone; only the
+    # winner's file remains.
+    library_files = [p for p in images_root.rglob("*.jpg") if p.is_file()]
+    assert library_files == [kept]
+
+
+def test_the_same_file_imported_twice_at_once_keeps_its_library_copy(
+    db_sync, layout, monkeypatch,
+):
+    """Both runs write the same library path, so the loser's copy IS the
+    winner's file and must not be deleted."""
+    import_root, _ = layout
+    src = import_root / "Alice" / "pic.jpg"
+    _jpeg(src)
+
+    winner = _worker(db_sync, layout).import_one(src)
+    kept = Path(db_sync.get(ImageRecord, winner.image_id).path)
+
+    loser = _worker(db_sync, layout)
+    _lose_the_race(loser, monkeypatch)
+    result = loser.import_one(src)
+
+    assert result.skip_reason == SkipReason.duplicate_hash
+    assert kept.is_file()
+
+
+def test_a_copy_attached_in_place_alongside_the_original_is_its_duplicate(
+    db_sync, layout, monkeypatch,
+):
+    _, images_root = layout
+    artist = Artist(name="Bea", slug="bea")
+    db_sync.add(artist)
+    db_sync.commit()
+    first = images_root / "bea" / "patreon" / "post1" / "pic.jpg"
+    _jpeg(first)
+    second = _copy(first, images_root / "bea" / "patreon" / "post2" / "pic.jpg")
+
+    winner = _worker(db_sync, layout).attach_in_place(first, artist=artist)
+    assert winner.status == "imported"
+
+    loser = _worker(db_sync, layout)
+    _lose_the_race(loser, monkeypatch)
+    result = loser.attach_in_place(second, artist=artist)
+
+    assert result.status == "skipped"
+    assert result.skip_reason == SkipReason.duplicate_hash
+    assert result.image_id == winner.image_id
+    monkeypatch.undo()
+    assert _records(db_sync) == 1
+    # Removing the loser's file is the caller's step (discard_after_attach).
+    assert first.is_file() and second.is_file()
+
+
+def test_an_archive_whose_member_breaks_the_transaction_is_still_stored(
+    db_sync, layout, monkeypatch,
+):
+    """A member whose import fails mid-flush leaves the transaction dead. The
+    member loop rolls it back, so the archive itself is still stored and the
+    import ends with a result instead of an exception."""
+    import_root, _ = layout
+    seed = _worker(db_sync, layout).import_one(_jpeg_at(import_root / "Cy" / "seed.jpg"))
+    seed_path = db_sync.get(ImageRecord, seed.image_id).path
+
+    arc = import_root / "Cy" / "set.zip"
+    with zipfile.ZipFile(arc, "w") as zf:
+        zf.writestr("a.jpg", _jpeg_bytes())
+
+    importer = _worker(db_sync, layout)
+
+    def breaks_the_transaction(*args, **kwargs):
+        importer.session.add(ImageRecord(
+            path=seed_path, sha256="f" * 64, size_bytes=1, mime="image/jpeg",
+            origin="imported_filesystem",
+        ))
+        importer.session.flush()  # refused: the path is taken
+
+    monkeypatch.setattr(importer, "_import_media", breaks_the_transaction)
+    result = importer.import_one(arc)
+
+    assert result.status == "attached"
+    stored = db_sync.execute(
+        select(func.count()).select_from(PostAttachment)
+    ).scalar_one()
+    assert stored == 1
+
+
+def _jpeg_at(path: Path) -> Path:
+    _jpeg(path)
+    return path
+
+
+def _jpeg_bytes() -> bytes:
+    buf = io.BytesIO()
+    Image.new("RGB", (64, 48), (200, 30, 30)).save(buf, "JPEG")
+    return buf.getvalue()

tests/test_library_scan.py

@@ -0,0 +1,123 @@
+"""Finding files in the artists' folders that the library has no record of, and
+importing them where they lie (services.library_scan + the `library` batch)."""
+
+import io
+import os
+import time
+
+import pytest
+from PIL import Image
+from sqlalchemy import select
+
+import backend.app.tasks.import_file as import_file
+from backend.app.models import Artist, ImageRecord, ImportSettings, ImportTask
+from backend.app.services import library_scan
+from backend.app.services.importer import Importer
+from backend.app.services.thumbnailer import Thumbnailer
+
+pytestmark = pytest.mark.integration
+
+
+def _jpeg(seed: int) -> bytes:
+    im = Image.new("RGB", (256, 256))
+    px = im.load()
+    for y in range(256):
+        for x in range(256):
+            px[x, y] = ((x * seed) % 256, (y * (seed + 1)) % 256, (x ^ y) % 256)
+    buf = io.BytesIO()
+    im.save(buf, "JPEG")
+    return buf.getvalue()
+
+
+def _write(path, data: bytes, *, age_days: float = 2):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(data)
+    t = time.time() - age_days * 86400
+    os.utime(path, (t, t))
+    return path
+
+
+@pytest.fixture
+def layout(db_sync, tmp_path):
+    db_sync.add(Artist(name="Libscan Maewix", slug="libscan-maewix"))
+    db_sync.commit()
+    post = tmp_path / "Libscan-Maewix" / "patreon" / "2026-05-01_123_Pack"
+    files = {
+        "missing": _write(post / "01_art.jpg", _jpeg(1)),
+        "owned": _write(post / "02_owned.jpg", _jpeg(2)),
+        "sidecar": _write(post / "01_art.json", b'{"category": "patreon", "id": "123"}'),
+        "staging": _write(post / "03_big.zip.part", b"partial"),
+        "recent": _write(post / "04_new.jpg", _jpeg(4), age_days=0),
+        "not_media": _write(post / "05_game.exe", b"MZ"),
+        "no_artist": _write(tmp_path / "libscan-nobody" / "x" / "a.jpg", _jpeg(5)),
+        "store": _write(tmp_path / "thumbs" / "abc" / "f.jpg", _jpeg(6)),
+    }
+    db_sync.add(ImageRecord(
+        path=str(files["owned"]), sha256="libscan-owned".ljust(64, "x"),
+        size_bytes=files["owned"].stat().st_size, mime="image/jpeg",
+        width=256, height=256, origin="downloaded", integrity_status="ok",
+    ))
+    db_sync.commit()
+    return tmp_path, files
+
+
+def test_finds_only_unrecorded_media_in_an_artists_folder(db_sync, layout):
+    """A differently-cased folder still matches its artist. Recorded files,
+    sidecars, staging files, the library's own stores, folders that match no
+    artist, files changed in the last day and non-media are not listed."""
+    root, files = layout
+    found, summary = library_scan.find_unimported(db_sync, root)
+
+    assert [u.path for u in found] == [files["missing"]]
+    assert found[0].artist_slug == "libscan-maewix"
+    assert found[0].task_type == "media"
+    assert summary["files"] == 1
+    assert summary["recent"] == 1
+    assert summary["not_media"] == 1
+    assert summary["no_artist"] == 1
+    assert summary["artists"][0]["slug"] == "libscan-maewix"
+    assert summary["artists"][0]["samples"] == [
+        "Libscan-Maewix/patreon/2026-05-01_123_Pack/01_art.jpg",
+    ]
+
+
+def test_queue_makes_a_library_batch_of_what_find_lists(db_sync, layout, monkeypatch):
+    """The import queued is exactly the previewed list."""
+    root, files = layout
+    queued = []
+    monkeypatch.setattr(import_file, "enqueue_import", lambda tid, tt: queued.append((tid, tt)))
+
+    summary = library_scan.queue_unimported(db_sync, root)
+
+    tasks = db_sync.execute(
+        select(ImportTask).where(ImportTask.batch_id == summary["batch_id"])
+    ).scalars().all()
+    assert [t.source_path for t in tasks] == [str(files["missing"])]
+    assert tasks[0].batch.scan_mode == "library"
+    assert queued == [(tasks[0].id, "media")]
+
+
+def test_a_library_import_attaches_in_place_and_removes_a_copy(db_sync, layout, monkeypatch):
+    """The file stays where it is and becomes the record's file. A byte-identical
+    copy found later is removed from disk, since the library already holds it."""
+    root, files = layout
+    monkeypatch.setattr(import_file, "IMAGES_ROOT", root)
+    importer = Importer(
+        session=db_sync, images_root=root, import_root=root,
+        thumbnailer=Thumbnailer(images_root=root),
+        settings=ImportSettings.load_sync(db_sync),
+    )
+
+    result = import_file._attach_where_it_lies(db_sync, importer, files["missing"])
+
+    assert result.status == "imported"
+    record = db_sync.get(ImageRecord, result.image_id)
+    assert record.path == str(files["missing"])
+    assert files["missing"].exists()
+    assert db_sync.get(Artist, record.artist_id).slug == "libscan-maewix"
+
+    copy = _write(files["missing"].parent / "09_copy.jpg", files["missing"].read_bytes())
+    again = import_file._attach_where_it_lies(db_sync, importer, copy)
+    assert again.status == "skipped"
+    assert not copy.exists()
+    assert files["missing"].exists()

tests/test_maintenance.py

@@ -26,26 +26,50 @@ def _make_batch(session) -> int:
     return batch.id
 
 
-def test_cleanup_orphaned_temp_files_removes_stale_only(tmp_path, monkeypatch):
+def test_cleanup_orphaned_temp_files_removes_stale_only(db_sync, tmp_path, monkeypatch):
     import os
 
+    from backend.app.models import ImageRecord, PostAttachment
     from backend.app.tasks import maintenance as m
 
     monkeypatch.setattr(m, "IMAGES_ROOT", tmp_path)
-    stale = tmp_path / "artist" / "img.jpg.part"      # killed download → orphan
-    stale.parent.mkdir(parents=True)
-    stale.write_bytes(b"x")
     old = datetime.now(UTC).timestamp() - 8 * 3600     # older than the 6h guard
-    os.utime(stale, (old, old))
-    fresh = tmp_path / "in_progress.jpg.partial"       # active download → keep
-    fresh.write_bytes(b"x")
-    keep = tmp_path / "real.jpg"                        # a real image → keep
-    keep.write_bytes(b"x")
 
-    assert m.cleanup_orphaned_temp_files() == 1
+    def staged(rel: str, *, aged: bool = True):
+        p = tmp_path / rel
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_bytes(b"x")
+        if aged:
+            os.utime(p, (old, old))
+        return p
+
+    stale = staged("artist/img.jpg.part")                 # killed download → orphan
+    mega = staged("artist/.megatmp.AbCd123")              # abandoned megatools stage
+    fresh = staged("in_progress.jpg.partial", aged=False)  # active download → keep
+    keep = tmp_path / "real.jpg"                          # a real image → keep
+    keep.write_bytes(b"x")
+    # Staging-named files a library row points at are library content.
+    attached = staged("attachments/ab/abc.part")
+    imaged = staged("artist/video.mp4.part")
+    db_sync.add(PostAttachment(
+        post_id=None, artist_id=None, sha256="tempsweep-att".ljust(64, "x"),
+        path=str(attached), original_filename="film.mp4.part", ext=".part",
+        size_bytes=1,
+    ))
+    db_sync.add(ImageRecord(
+        path=str(imaged), sha256="tempsweep-img".ljust(64, "x"), size_bytes=1,
+        mime="video/mp4", width=1, height=1, origin="downloaded",
+        integrity_status="ok",
+    ))
+    db_sync.commit()
+
+    assert m.cleanup_orphaned_temp_files() == 2
     assert not stale.exists()
+    assert not mega.exists()
     assert fresh.exists()
     assert keep.exists()
+    assert attached.exists()
+    assert imaged.exists()
 
 
 def test_recover_interrupted_only_old(db_sync, monkeypatch):
@@ -98,7 +122,10 @@ def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypat
     Banked 2026-05-25 after operator hit 5490 stuck pending rows.
     """
     from backend.app.tasks import import_file
+    from backend.app.tasks import maintenance as m
     monkeypatch.setattr(import_file.import_media_file, "delay", lambda *_: None)
+    # The broker's import queue is empty, so a stale 'queued' row's message is gone.
+    monkeypatch.setattr(m, "_import_queue_depth", lambda: 0)
 
     batch_id = _make_batch(db_sync)
     now = datetime.now(UTC)
@@ -146,6 +173,42 @@ def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypat
     assert "orphan" in (stale_pending.error or "")
 
 
+@pytest.mark.parametrize("depth", [7, None])
+def test_recover_interrupted_keeps_queued_rows_while_the_import_queue_holds_work(
+    db_sync, monkeypatch, depth,
+):
+    """A queued row in a batch bigger than half an hour of work is waiting its
+    turn, not orphaned: while the import queue holds messages (or can't be
+    read), only never-enqueued 'pending' rows are swept."""
+    from sqlalchemy import update as _upd
+
+    from backend.app.tasks import maintenance as m
+
+    monkeypatch.setattr(m, "_import_queue_depth", lambda: depth)
+    batch_id = _make_batch(db_sync)
+    waiting = ImportTask(
+        batch_id=batch_id, source_path="/import/waiting.jpg", task_type="media",
+        status="queued",
+    )
+    never_sent = ImportTask(
+        batch_id=batch_id, source_path="/import/never.jpg", task_type="media",
+        status="pending",
+    )
+    db_sync.add_all([waiting, never_sent])
+    db_sync.flush()
+    db_sync.execute(
+        _upd(ImportTask).where(ImportTask.id.in_([waiting.id, never_sent.id]))
+        .values(created_at=datetime.now(UTC) - timedelta(hours=2))
+    )
+    db_sync.commit()
+
+    assert m.recover_interrupted_tasks.apply().get() == 1
+    db_sync.refresh(waiting)
+    db_sync.refresh(never_sent)
+    assert waiting.status == "queued"
+    assert never_sent.status == "failed"
+
+
 def test_recover_interrupted_handles_both_stuck_and_orphans(db_sync, monkeypatch):
     """One sweep tick handles both 'processing' crashes AND
     'pending'/'queued' orphans in a single pass."""
@@ -885,3 +948,14 @@ def test_reclaim_attachments_stuck_threshold_exceeds_hard_time_limit():
         "backend.app.tasks.admin.reclaim_orphaned_attachments_task"
     ]
     assert override >= hard_minutes
+
+
+def test_import_unimported_stuck_threshold_exceeds_hard_time_limit():
+    """The same invariant for the walk that imports files on disk."""
+    from backend.app.tasks.admin import import_unimported_files_task
+    from backend.app.tasks.maintenance import TASK_STUCK_THRESHOLD_MINUTES
+
+    override = TASK_STUCK_THRESHOLD_MINUTES[
+        "backend.app.tasks.admin.import_unimported_files_task"
+    ]
+    assert override >= import_unimported_files_task.time_limit / 60

tests/test_patreon_ingester.py

@@ -984,6 +984,31 @@ async def test_recapture_imports_a_seen_file_nothing_imported(
     assert downloader.download_calls == 0
 
 
+@pytest.mark.asyncio
+async def test_files_phase_3_leaves_unimported_stay_unmarked(
+    source_id, sync_engine, tmp_path,
+):
+    """Phase 3 can stop importing at its deadline. Each written file names the
+    page it came from, so the next chunk can resume there; the files left keep
+    no ledger mark, so that walk meets them on disk and imports them."""
+    m1, m2 = _media("p1", 1), _media("p2", 1)
+    pages = [("CUR1", [("p1", [m1])]), ("CUR2", [("p2", [m2])])]
+    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
+    result = ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
+                     url="https://patreon.com/ingest", mode="backfill")
+    assert result.written_cursors == ["CUR1", "CUR2"]
+
+    result.mark_seen_after_import(skip_paths={str(tmp_path / "p2_1.jpg")})
+    assert _count_ledger(sync_engine, source_id) == 3   # m1 and both post records
+
+    ing2 = _ingester(sync_engine, tmp_path, _FakeClient(pages),
+                     _FakeDownloader(tmp_path, on_disk={_ledger_key(m1), _ledger_key(m2)}))
+    again = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
+                     url="https://patreon.com/ingest", mode="backfill")
+    assert again.written_paths == [str(tmp_path / "p2_1.jpg")]
+    assert "on disk but never imported: p2_1.jpg" in again.stdout
+
+
 # --- dead-letter ledger (plan #705 #7) ------------------------------------
 
 

tests/test_post_feed_service.py

@@ -283,6 +283,27 @@ async def test_scroll_item_shape_minimal(db):
     assert "description_full" not in item
 
 
+@pytest.mark.asyncio
+async def test_a_discord_post_names_its_channel(db):
+    """#4481: the card says which channel a message came from. Read from the
+    record's `channel`; never on another platform, even with the same key."""
+    artist = await _seed_artist(db, "todding-ch")
+    dsrc = await _seed_source(db, artist.id, "discord", "https://discord.com/channels/1/2")
+    psrc = await _seed_source(db, artist.id, "patreon", "https://p/todding-ch")
+    now = datetime.now(UTC)
+    d = await _seed_post(db, dsrc.id, external_id="D1", post_date=now)
+    d.raw_metadata = {"category": "discord", "channel": "banana-land"}
+    blank = await _seed_post(db, dsrc.id, external_id="D2", post_date=now)
+    blank.raw_metadata = {"category": "discord", "channel": "  "}
+    p = await _seed_post(db, psrc.id, external_id="P1", post_date=now)
+    p.raw_metadata = {"channel": "not-discord"}
+    await db.commit()
+
+    page = await PostFeedService(db).scroll(cursor=None, limit=10, artist_id=artist.id)
+    by_id = {it["external_post_id"]: it["channel"] for it in page["items"]}
+    assert by_id == {"D1": "banana-land", "D2": None, "P1": None}
+
+
 @pytest.mark.asyncio
 async def test_scroll_surfaces_translation_fields(db):
     # #143: a translated post exposes the translated title/description + source

tests/test_task_run_signals.py

@@ -20,10 +20,9 @@ pytestmark = pytest.mark.integration
 
 
 @pytest.fixture(autouse=True)
-def _eager_celery(monkeypatch):
+def _eager_celery(celery_conf):
     """task_always_eager so signals fire in-process during the test."""
-    monkeypatch.setattr(celery.conf, "task_always_eager", True)
-    monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
+    celery_conf(task_always_eager=True, task_eager_propagates=False)
 
 
 @pytest.mark.asyncio

tests/test_tasks_admin.py

@@ -15,9 +15,8 @@ pytestmark = pytest.mark.integration
 
 
 @pytest.fixture(autouse=True)
-def _eager_celery(monkeypatch):
-    monkeypatch.setattr(celery.conf, "task_always_eager", True)
-    monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
+def _eager_celery(celery_conf):
+    celery_conf(task_always_eager=True, task_eager_propagates=False)
 
 
 @pytest.fixture(autouse=True)

tests/test_tasks_backup.py

@@ -10,16 +10,14 @@ from pathlib import Path
 import pytest
 from sqlalchemy import select
 
-from backend.app.celery_app import celery
 from backend.app.models import BackupRun, ImportSettings
 
 pytestmark = pytest.mark.integration
 
 
 @pytest.fixture(autouse=True)
-def _eager_celery(monkeypatch):
-    monkeypatch.setattr(celery.conf, "task_always_eager", True)
-    monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
+def _eager_celery(celery_conf):
+    celery_conf(task_always_eager=True, task_eager_propagates=False)
 
 
 @pytest.fixture(autouse=True)
@@ -35,7 +33,7 @@ def fake_subprocess_and_images_root(monkeypatch, tmp_path):
         stderr = b""
 
     def _sentinel(cmd):
-        if cmd[0] == "pg_dump":
+        if "pg_dump" in cmd:
             i = cmd.index("-f")
             Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
         elif cmd[0] == "tar" and "-cf" in cmd:
@@ -130,6 +128,63 @@ async def test_backup_db_task_persists_tag(db_sync):
     assert tag == "pre-cutover"
 
 
+@pytest.fixture
+def archives(monkeypatch):
+    """An off-site destination is set up; the dump itself is faked."""
+    from backend.app.services.backup_s3 import S3Target
+    target = S3Target("http://archives.example:8081", "curator", "AK", "SK")
+    monkeypatch.setattr(
+        "backend.app.services.backup_destination.saved_target", lambda settings: target,
+    )
+    sent = []
+
+    def _offsite(*, db_url, target, limit_mbps):
+        sent.append({"target": target, "limit_mbps": limit_mbps})
+        return {
+            "kind": "db", "ts": "20261107T030000Z",
+            "key": "database/fabledcurator-20261107T030000Z.dump",
+            "bucket": target.bucket, "endpoint": target.endpoint,
+            "parts": 3, "size_bytes": 1234,
+        }
+    monkeypatch.setattr("backend.app.services.backup_service.backup_db_offsite", _offsite)
+    return sent
+
+
+@pytest.mark.asyncio
+async def test_with_a_destination_the_dump_goes_off_site_and_nothing_is_written_here(
+    db_sync, archives, tmp_path,
+):
+    from backend.app.tasks.backup import backup_db_task
+    run_id = backup_db_task.delay(triggered_by="nightly").get()["backup_run_id"]
+    row = db_sync.get(BackupRun, run_id)
+    assert row.status == "ok"
+    assert row.sql_path is None
+    assert row.size_bytes == 1234
+    assert row.manifest["offsite"] == {
+        "endpoint": "http://archives.example:8081", "bucket": "curator",
+        "key": "database/fabledcurator-20261107T030000Z.dump", "parts": 3,
+    }
+    assert archives[0]["limit_mbps"] == 200        # the setting's default
+    assert not (tmp_path / "_backups").exists()
+
+
+@pytest.mark.asyncio
+async def test_an_off_site_dump_that_fails_is_recorded_with_why(db_sync, monkeypatch, archives):
+    from backend.app.services.backup_service import DumpFailed
+    from backend.app.tasks.backup import backup_db_task
+
+    def _fails(**kwargs):
+        raise DumpFailed("pg_dump: error: connection lost")
+    monkeypatch.setattr("backend.app.services.backup_service.backup_db_offsite", _fails)
+    with pytest.raises(DumpFailed):
+        backup_db_task.delay().get()
+    row = db_sync.execute(
+        select(BackupRun).where(BackupRun.kind == "db").order_by(BackupRun.id.desc()).limit(1)
+    ).scalar_one()
+    assert row.status == "error"
+    assert row.error == "DumpFailed: pg_dump: error: connection lost"
+
+
 # --- backup_images_task ---------------------------------------------
 
 

tests/test_tasks_library_audit.py

@@ -9,7 +9,6 @@ from PIL import Image
 from sqlalchemy import select
 
 import backend.app.tasks.library_audit  # noqa: F401 — celery registration
-from backend.app import celery_app
 from backend.app.models import ImageRecord, LibraryAuditRun
 
 pytestmark = pytest.mark.integration
@@ -37,7 +36,7 @@ def _mk_image(db_sync, tmp_path, *, mode, color, name):
 
 
 def test_scan_library_for_rule_populates_matched_ids_for_transparency(
-    db_sync, tmp_path, monkeypatch,
+    db_sync, tmp_path, celery_conf,
 ):
     transparent_rec, _ = _mk_image(
         db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="trans.png",
@@ -53,10 +52,10 @@ def test_scan_library_for_rule_populates_matched_ids_for_transparency(
     db_sync.commit()
     audit_id = audit.id
 
-    monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True)
+    celery_conf(task_always_eager=True)
     from backend.app.tasks.library_audit import scan_library_for_rule
     scan_library_for_rule.run(audit_id)
-    monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False)
+    celery_conf(task_always_eager=False)
 
     matched = db_sync.execute(
         select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id)
@@ -124,7 +123,7 @@ def test_scan_resumes_and_accumulates_matched(db_sync, tmp_path, monkeypatch):
 
 
 def test_scan_library_for_rule_skips_missing_files_gracefully(
-    db_sync, tmp_path, monkeypatch,
+    db_sync, tmp_path, celery_conf,
 ):
     rec, path = _mk_image(
         db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="ghost.png",
@@ -138,10 +137,10 @@ def test_scan_library_for_rule_skips_missing_files_gracefully(
     db_sync.commit()
     audit_id = audit.id
 
-    monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True)
+    celery_conf(task_always_eager=True)
     from backend.app.tasks.library_audit import scan_library_for_rule
     scan_library_for_rule.run(audit_id)
-    monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False)
+    celery_conf(task_always_eager=False)
 
     status = db_sync.execute(
         select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)