Merge pull request 'Release: dev → main (Discord poster picker, chunk import deadline, CI repairs)' (#260) from dev into main

ci: buildx driver, registry layer cache, and a force_build hatch (milestone 326 steps 1-3)

.forgejo/workflows/build.yml

@@ -202,8 +202,7 @@ env:
 # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
 #   - write:package, read:package  (for docker push to git.fabledsword.com)
 #   - write:release                (for ext-<version> release asset cache)
-#   - write:issue                  (`unattended-failure` opens an issue)
-#   - write:repository             (`refresh-dev` dispatches this workflow)
+#   - write:issue                  (for future issue-management automation)
 # The injected GITHUB_TOKEN cannot be used — it lacks write:package.
 
 jobs:
@@ -1447,11 +1446,8 @@ jobs:
           # `no-cache: true`, which is a much larger version of the same
           # churn #3265 is about.
           #
-          # Only on the refresh and on a forced build. An ordinary push wants
-          # the cached base. A forced build pulls because forcing is how the
-          # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
-          # keeps the old base layers would be the same image again.
-          pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
+          # Only on the schedule. An ordinary push wants the cached base.
+          pull: ${{ env.IS_REFRESH == 'true' }}
           # ONE tag, the channel's. Every other tag is written by the step
           # below, registry-side. buildx here pushes the first tag to the
           # registry and then re-pushes the rest through the DOCKER driver,
@@ -2206,121 +2202,6 @@ jobs:
             exit 1
           fi
           echo "promote: $MOVED tag(s) written"
-
-  # The refresh rebuilds `:dev` as well as `:latest`.
-  #
-  # The refresh run itself builds main and publishes `:latest`. `:dev` needs a
-  # build of dev's own source with FC_CHANNEL=dev baked in, so copying main's
-  # refreshed bytes onto `:dev` is not the same thing: the image would report
-  # itself as main, and the reuse check would then keep it that way. Instead
-  # this job dispatches an ordinary dev run with `force_build`. That run goes
-  # through the lanes, the smoke and `promote` like any push, and a forced
-  # build pulls fresh bases (see `pull:` in both build jobs).
-  #
-  # Without this, an artifact whose source is quiet on dev (the agent, most
-  # weeks) keeps the base layers of its last build indefinitely, because
-  # reuse skips its build on every push.
-  #
-  # The dispatched run shares this run's concurrency group (both are on the
-  # dev ref), so it queues until this run has finished. It runs whatever
-  # `promote` did: a failed `:latest` refresh is no reason to leave `:dev`
-  # stale, and `unattended-failure` reports the failure either way.
-  refresh-dev:
-    needs: [promote]
-    if: >-
-      !cancelled() && (github.event_name == 'schedule'
-      || format('{0}', github.event.inputs.refresh) == 'true')
-    runs-on: python-ci
-    container:
-      image: git.fabledsword.com/bvandeusen/ci-python:3.14
-    steps:
-      - name: Dispatch a forced dev build
-        env:
-          TOKEN: ${{ secrets.RELEASE_TOKEN }}
-          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
-        run: |
-          set -eu
-          CODE=$(curl -sS --max-time 30 -o response.txt -w '%{http_code}' -X POST \
-            -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
-            -d '{"ref":"dev","inputs":{"force_build":"true"}}' \
-            "$API/actions/workflows/build.yml/dispatches")
-          echo "refresh-dev: dispatch answered HTTP $CODE"
-          case "$CODE" in
-            2??) echo "refresh-dev: a forced dev run is queued behind this one" ;;
-            *)
-              cat response.txt >&2
-              echo "refresh-dev: the dispatch was refused. A 403 means RELEASE_TOKEN" >&2
-              echo "refresh-dev: lacks write:repository (see the scope list at the top)." >&2
-              exit 1 ;;
-          esac
-
-  # A run nobody is watching reports its own failure.
-  #
-  # A push run is watched by whoever pushed it. The weekly refresh and the dev
-  # run it dispatches are not, and a red run on the Actions page alone goes
-  # unseen. So any failed run that is not a push or a pull request opens an
-  # issue, or comments on the one already open, naming the run and the jobs
-  # that failed. A manual dispatch that fails reports here too; closing the
-  # issue once it is dealt with is what keeps the next failure visible.
-  #
-  # The issue is written with RELEASE_TOKEN, so it is authored by the token's
-  # owner, and the forge does not notify people of their own actions. It
-  # shows in the repository's open issues rather than as an email.
-  unattended-failure:
-    needs: [lint, extension-version, backend-lint-and-test, frontend-build,
-            extension-test, integration, sign-extension, build-web, smoke-web,
-            build-agent, promote, refresh-dev]
-    if: >-
-      failure() && github.event_name != 'push'
-      && github.event_name != 'pull_request'
-    runs-on: python-ci
-    container:
-      image: git.fabledsword.com/bvandeusen/ci-python:3.14
-    steps:
-      - name: Open or update the failure issue
-        env:
-          TOKEN: ${{ secrets.RELEASE_TOKEN }}
-          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          EVENT: ${{ github.event_name }}
-          REF: ${{ github.ref_name }}
-          NEEDS: ${{ toJSON(needs) }}
-        run: |
-          python3 - <<'EOF'
-          import json, os, urllib.parse, urllib.request
-
-          TITLE = "CI: an unattended build.yml run failed"
-          api, token = os.environ["API"], os.environ["TOKEN"]
-
-          def call(method, path, body=None):
-              req = urllib.request.Request(
-                  api + path, method=method,
-                  data=json.dumps(body).encode() if body is not None else None,
-                  headers={"Authorization": f"token {token}",
-                           "Content-Type": "application/json"})
-              with urllib.request.urlopen(req, timeout=30) as resp:
-                  return json.load(resp)
-
-          needs = json.loads(os.environ["NEEDS"])
-          failed = sorted(j for j, v in needs.items() if v.get("result") == "failure")
-          text = (f"Run: {os.environ['RUN_URL']}\n"
-                  f"Trigger: `{os.environ['EVENT']}` on `{os.environ['REF']}`\n"
-                  f"Failed jobs: {', '.join(f'`{j}`' for j in failed) or '(none reported)'}\n")
-
-          q = urllib.parse.urlencode({"state": "open", "type": "issues", "q": TITLE})
-          existing = [i for i in call("GET", f"/issues?{q}") if i["title"] == TITLE]
-          if existing:
-              n = existing[0]["number"]
-              call("POST", f"/issues/{n}/comments", {"body": text})
-              print(f"unattended-failure: commented on open issue #{n}")
-          else:
-              body = ("A build.yml run that nobody watches failed. Read the run's "
-                      "failing job logs; close this issue once it is fixed, so the "
-                      "next failure opens a new one.\n\n" + text)
-              n = call("POST", "/issues", {"title": TITLE, "body": body})["number"]
-              print(f"unattended-failure: opened issue #{n}")
-          EOF
-
   build-agent:
     # THE GATE (2026-09-23). Every lane above must have PASSED before this job
     # exists at all — so a red suite does not produce an image, let alone push
@@ -2659,11 +2540,8 @@ jobs:
           # `no-cache: true`, which is a much larger version of the same
           # churn #3265 is about.
           #
-          # Only on the refresh and on a forced build. An ordinary push wants
-          # the cached base. A forced build pulls because forcing is how the
-          # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
-          # keeps the old base layers would be the same image again.
-          pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
+          # Only on the schedule. An ordinary push wants the cached base.
+          pull: ${{ env.IS_REFRESH == 'true' }}
           # ONE tag, the channel's. Every other tag is written by the step
           # below, registry-side. buildx here pushes the first tag to the
           # registry and then re-pushes the rest through the DOCKER driver,

alembic/versions/0117_external_link_started_at.py

@@ -1,33 +0,0 @@
-"""external_link.started_at: when a fetch claimed the link.
-
-A worker killed mid-fetch (the Celery hard limit, an OOM, a restart) never
-reaches the code that moves its link out of 'downloading'. The external-link
-sweep returns such a link to the retry path once it has been 'downloading'
-longer than the task's hard limit, which needs the claim time.
-
-A row in 'downloading' with no claim time is treated as stale: no worker
-survives the deploy that adds this column.
-
-Revision ID: 0117
-Revises: 0116
-Create Date: 2026-11-05
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-revision = "0117"
-down_revision = "0116"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "external_link",
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("external_link", "started_at")

alembic/versions/0118_external_link_recoveries.py

@@ -1,32 +0,0 @@
-"""external_link.recoveries: how many times recovery has retried a dead link.
-
-The daily recovery gives a dead link one more attempt, spaced further apart
-each time, and leaves it dead once the count reaches its limit. A link that
-can never succeed is then downloaded a handful of times in total, not every
-day.
-
-Revision ID: 0118
-Revises: 0117
-Create Date: 2026-11-05
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-revision = "0118"
-down_revision = "0117"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "external_link",
-        sa.Column(
-            "recoveries", sa.Integer(), nullable=False, server_default=sa.text("0")
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("external_link", "recoveries")

alembic/versions/0119_backup_s3_destination.py

@@ -1,43 +0,0 @@
-"""import_settings: the off-site backup destination (an S3 bucket).
-
-Address, bucket, access key and region are plain settings; the secret key is
-stored sealed with the credential key. All empty means no destination is set
-up, which is every existing install's state.
-
-Revision ID: 0119
-Revises: 0118
-Create Date: 2026-11-06
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-revision = "0119"
-down_revision = "0118"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column("import_settings", sa.Column("backup_s3_endpoint", sa.Text(), nullable=True))
-    op.add_column(
-        "import_settings", sa.Column("backup_s3_bucket", sa.String(255), nullable=True),
-    )
-    op.add_column("import_settings", sa.Column("backup_s3_access_key", sa.Text(), nullable=True))
-    op.add_column(
-        "import_settings", sa.Column("backup_s3_secret_sealed", sa.Text(), nullable=True),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "backup_s3_region", sa.String(64), nullable=False, server_default="us-east-1",
-        ),
-    )
-
-
-def downgrade() -> None:
-    for column in (
-        "backup_s3_region", "backup_s3_secret_sealed", "backup_s3_access_key",
-        "backup_s3_bucket", "backup_s3_endpoint",
-    ):
-        op.drop_column("import_settings", column)

alembic/versions/0120_backup_upload_limit.py

@@ -1,30 +0,0 @@
-"""import_settings: a bandwidth limit for uploads to the off-site backup bucket.
-
-In megabits per second; 0 means no limit. Every install starts at 200 Mbit/s,
-which leaves room on a gigabit link for everything else the server does.
-
-Revision ID: 0120
-Revises: 0119
-Create Date: 2026-11-07
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-revision = "0120"
-down_revision = "0119"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "backup_upload_limit_mbps", sa.Integer(), nullable=False, server_default="200",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "backup_upload_limit_mbps")

alembic/versions/0121_backup_blob.py

@@ -1,46 +0,0 @@
-"""backup_blob: which library files are already in the off-site bucket.
-
-One row per (destination, sha256) sent. The media backup's work list is every
-image and attachment sha with no row for the current destination, so a run is
-incremental and resumable by construction, and pointing backups at a new
-bucket starts that bucket's copy from the beginning.
-
-backup_run.last_progress_at is the heartbeat of a run that spans hours; the
-stall sweep measures a media run's silence from it, not from its start.
-
-Revision ID: 0121
-Revises: 0120
-Create Date: 2026-11-07
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-revision = "0121"
-down_revision = "0120"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "backup_blob",
-        sa.Column("target", sa.Text(), nullable=False),
-        sa.Column("sha256", sa.String(64), nullable=False),
-        sa.Column("key", sa.Text(), nullable=False),
-        sa.Column("size_bytes", sa.BigInteger(), nullable=False),
-        sa.Column(
-            "uploaded_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.text("now()"),
-        ),
-        sa.PrimaryKeyConstraint("target", "sha256", name=op.f("pk_backup_blob")),
-    )
-    op.add_column(
-        "backup_run",
-        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("backup_run", "last_progress_at")
-    op.drop_table("backup_blob")

backend/app/api/admin.py

@@ -475,21 +475,6 @@ async def trigger_reclaim_attachments():
     return _queued(async_result)
 
 
-@admin_bp.route("/maintenance/import-unimported", methods=["POST"])
-async def trigger_import_unimported():
-    """Import files in the artists' folders that the library has no record of.
-    Body {"dry_run": bool}; dry_run (the DEFAULT) lists what was found, per
-    artist, and changes nothing. dry_run=false queues them as an import batch
-    that attaches each file where it lies. Returns the Celery task id — poll
-    /maintenance/task-result/<id> for the summary."""
-    from ..tasks.admin import import_unimported_files_task
-
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", True))
-    async_result = import_unimported_files_task.delay(dry_run=dry_run)
-    return _queued(async_result)
-
-
 @admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
 async def trigger_repair_discord_downloads():
     """Clean re-download of the Discord files broken by the `None` naming

backend/app/api/system_backup.py

@@ -20,7 +20,7 @@ system_backup_bp = Blueprint(
     "system_backup", __name__, url_prefix="/api/system/backup",
 )
 
-_KINDS = frozenset({"db", "images", "media"})
+_KINDS = frozenset({"db", "images"})
 _TAG_MAX_LEN = 64
 _BACKUP_SETTINGS_FIELDS = (
     "backup_db_nightly_enabled",
@@ -49,9 +49,6 @@ def _row_to_dict(r: BackupRun) -> dict:
         "error": r.error,
         "restored_from_id": r.restored_from_id,
         "manifest": r.manifest or {},
-        # Where an off-site dump went ({endpoint, bucket, key}); None for a
-        # backup kept on this server.
-        "offsite": (r.manifest or {}).get("offsite"),
     }
 
 
@@ -110,38 +107,6 @@ async def trigger_images_backup():
     return jsonify({"status": "dispatched"}), 202
 
 
-@system_backup_bp.route("/media", methods=["POST"])
-async def trigger_media_backup():
-    """Send the images and attachments not yet in the off-site bucket. Refused
-    when no destination is set up; a run already going is left to finish."""
-    from ..services import backup_destination as dest
-
-    async with get_session() as session:
-        row = await ImportSettings.load(session)
-        if not dest.describe(row)["configured"]:
-            return _bad("no_destination",
-                        detail="set up an off-site destination before sending files")
-    from ..tasks.backup import backup_media_task
-    backup_media_task.delay(triggered_by="manual")
-    return jsonify({"status": "dispatched"}), 202
-
-
-@system_backup_bp.route("/media", methods=["GET"])
-async def media_coverage():
-    """How much of the library is in the off-site bucket: files and bytes,
-    sent and total, each file counted once however many posts hold it."""
-    from ..services import backup_destination as dest
-    from ..services import backup_media
-
-    async with get_session() as session:
-        row = await ImportSettings.load(session)
-        f = dest.fields_of(row)
-        configured = dest.describe(row)["configured"]
-        tid = backup_media.destination_id(f.endpoint, f.bucket) if configured else None
-        counts = await session.run_sync(lambda s: backup_media.coverage(s, tid))
-    return jsonify({"configured": configured, **counts})
-
-
 @system_backup_bp.route("/runs", methods=["GET"])
 async def list_runs():
     try:
@@ -215,12 +180,6 @@ async def trigger_restore(run_id: int):
                 "not_restorable",
                 detail=f"source backup status={row.status!r}; only 'ok' rows are restorable",
             )
-        if (row.manifest or {}).get("offsite"):
-            return _bad(
-                "not_restorable",
-                detail=("this dump is kept in the off-site bucket, not on this server, "
-                        "so it can't be restored from here"),
-            )
         expected = f"restore-{row.kind}-{row.id}"
         if supplied != expected:
             return _bad(
@@ -277,118 +236,6 @@ async def get_settings():
     })
 
 
-@system_backup_bp.route("/destination", methods=["GET"])
-async def get_destination():
-    """The off-site destination as the settings screen shows it. The secret
-    key is never returned, only whether one is saved."""
-    from ..services import backup_destination as dest
-
-    async with get_session() as session:
-        row = await ImportSettings.load(session)
-        return jsonify(dest.describe(row))
-
-
-@system_backup_bp.route("/destination", methods=["PATCH"])
-async def patch_destination():
-    """Save the destination. Body {endpoint, bucket, access_key, region,
-    secret_key, upload_limit_mbps}; an empty field clears it, and a missing
-    upload_limit_mbps keeps the saved limit. A missing or empty secret_key
-    keeps the saved one, so the screen can save the other fields without the
-    secret being typed again."""
-    from ..services import backup_destination as dest
-
-    body = await request.get_json(silent=True)
-    if not isinstance(body, dict):
-        return _bad("invalid_body", detail="body must be a JSON object")
-    fields, why = dest.clean_fields(body)
-    if fields is None:
-        return _bad("invalid_destination", detail=why)
-    secret = body.get("secret_key")
-    if secret is not None and not isinstance(secret, str):
-        return _bad("invalid_destination", detail="secret_key must be text")
-    limit = body.get("upload_limit_mbps")
-    if limit is not None and (
-        not isinstance(limit, int) or isinstance(limit, bool) or not 0 <= limit <= 100_000
-    ):
-        return _bad("invalid_destination",
-                    detail="upload_limit_mbps must be a whole number from 0 (no limit) up")
-
-    async with get_session() as session:
-        row = await ImportSettings.load(session)
-        if limit is not None:
-            row.backup_upload_limit_mbps = limit
-        row.backup_s3_endpoint = fields.endpoint
-        row.backup_s3_bucket = fields.bucket
-        row.backup_s3_access_key = fields.access_key
-        row.backup_s3_region = fields.region
-        if secret and secret.strip():
-            row.backup_s3_secret_sealed = dest.seal(secret.strip())
-        await session.commit()
-        return jsonify(dest.describe(row))
-
-
-@system_backup_bp.route("/destination", methods=["DELETE"])
-async def delete_destination():
-    """Forget the destination, secret included. What was already sent stays in
-    the bucket: FabledCurator never deletes there."""
-    from ..services import backup_destination as dest
-
-    async with get_session() as session:
-        row = await ImportSettings.load(session)
-        row.backup_s3_endpoint = None
-        row.backup_s3_bucket = None
-        row.backup_s3_access_key = None
-        row.backup_s3_secret_sealed = None
-        row.backup_s3_region = dest.DEFAULT_REGION
-        await session.commit()
-        return jsonify(dest.describe(row))
-
-
-@system_backup_bp.route("/destination/test", methods=["POST"])
-async def try_destination():
-    """Reach the bucket with the fields in the body (the form as typed, saved
-    or not); a missing secret_key uses the saved one. Lists the bucket and
-    writes nothing. Answers {ok, message} either way: a destination that
-    can't be reached is a result to show, not an API error."""
-    import asyncio
-
-    from ..services import backup_destination as dest
-    from ..services.backup_s3 import S3Error
-    from ..services.credential_crypto import InvalidCredentialBlob
-
-    body = await request.get_json(silent=True) or {}
-    if not isinstance(body, dict):
-        return _bad("invalid_body", detail="body must be a JSON object")
-    fields, why = dest.clean_fields(body)
-    if fields is None:
-        return jsonify({"ok": False, "message": f"Not tested: {why}."})
-    secret = body.get("secret_key")
-    secret = secret.strip() if isinstance(secret, str) else ""
-    if not secret:
-        async with get_session() as session:
-            row = await ImportSettings.load(session)
-            sealed = row.backup_s3_secret_sealed
-        if sealed:
-            try:
-                secret = dest.unseal(sealed)
-            except InvalidCredentialBlob:
-                return jsonify({"ok": False, "message": (
-                    "Not tested: the saved secret key can't be opened on this server. "
-                    "Enter the secret key again."
-                )})
-    target = dest.target_for(fields, secret or None)
-    if target is None:
-        missing = [name for name, v in (
-            ("address", fields.endpoint), ("bucket", fields.bucket),
-            ("access key", fields.access_key), ("secret key", secret),
-        ) if not v]
-        return jsonify({"ok": False, "message": f"Not tested: fill in the {', '.join(missing)}."})
-    try:
-        return jsonify(await asyncio.to_thread(dest.check_target, target))
-    except S3Error as exc:
-        return jsonify({"ok": False, "message": str(exc)})
-
-
 @system_backup_bp.route("/settings", methods=["PATCH"])
 async def patch_settings():
     body = await request.get_json(silent=True)

backend/app/celery_app.py

@@ -302,10 +302,6 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.backup.prune_backups",
                 "schedule": 86400.0,  # daily
             },
-            "reconcile-media-backup": {
-                "task": "backend.app.tasks.backup.reconcile_media_backup",
-                "schedule": 604800.0,  # weekly: list the bucket, re-send what's missing
-            },
             # Audit 2026-06-02 — three new per-entity recovery sweeps.
             # Each runs every 5 min like the other recover_stalled_*
             # sweeps; each is a no-op when nothing is stuck.

backend/app/models/__init__.py

@@ -4,7 +4,6 @@ from .app_setting import AppSetting
 from .artist import Artist
 from .artist_membership_suggestion import ArtistMembershipSuggestion
 from .artist_visit import ArtistVisit
-from .backup_blob import BackupBlob
 from .backup_run import BackupRun
 from .base import Base
 from .character_prototype import CcipPrototypeState, CharacterPrototype
@@ -56,7 +55,6 @@ __all__ = [
     "Artist",
     "ArtistMembershipSuggestion",
     "ArtistVisit",
-    "BackupBlob",
     "BackupRun",
     "Source",
     "Credential",

backend/app/models/backup_blob.py

@@ -1,26 +0,0 @@
-"""backup_blob: a library file already sent to the off-site bucket.
-
-One row per (destination, sha256). `target` names the destination as
-"<endpoint>/<bucket>", so pointing backups at another bucket leaves these rows
-inert and that bucket's copy starts from the beginning. `key` is where the
-file sits in the bucket (`media/ab/cd/<sha256><ext>`).
-"""
-
-from datetime import datetime
-
-from sqlalchemy import BigInteger, DateTime, String, Text, func
-from sqlalchemy.orm import Mapped, mapped_column
-
-from .base import Base
-
-
-class BackupBlob(Base):
-    __tablename__ = "backup_blob"
-
-    target: Mapped[str] = mapped_column(Text, primary_key=True)
-    sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
-    key: Mapped[str] = mapped_column(Text, nullable=False)
-    size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
-    uploaded_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now(),
-    )

backend/app/models/backup_run.py

@@ -1,7 +1,6 @@
 """FC-3h: backup_run — operator-facing artifact record for a backup run.
 
-One row per backup attempt (kind='db', 'images', or 'media' for the off-site
-upload of images and attachments, which spans many task chunks). Lifecycle
+One row per backup attempt (kind='db' or 'images'). Lifecycle
 tracking (started_at/finished_at/duration_ms/exception text) lives
 in task_run from FC-3i — this row records artifact metadata: file
 paths, sizes, tag (retention protection), and restore lineage via
@@ -63,11 +62,6 @@ class BackupRun(Base):
     )
     # Self-referential FK, unindexed until 0089 (#3300): SET NULL has to find
     # the rows pointing at a deleted run before it can null them.
-    # Heartbeat of a run that spans hours (the media backup): set as it makes
-    # progress, and what the stall sweep measures its silence from.
-    last_progress_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True,
-    )
     restored_from_id: Mapped[int | None] = mapped_column(
         ForeignKey("backup_run.id", ondelete="SET NULL"),
         nullable=True, index=True,

backend/app/models/external_link.py

@@ -80,23 +80,12 @@ class ExternalLink(Base):
         Integer, nullable=False, server_default=text("0")
     )
     last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
-    # Times the daily recovery has given this link another attempt after it
-    # went dead. Recovery stops at a limit, so a link that can never succeed
-    # isn't downloaded forever.
-    recoveries: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
     attachment_id: Mapped[int | None] = mapped_column(
         ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
     )
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-    # When the current fetch claimed the link ('downloading'). The sweep uses it
-    # to recover a link whose worker died mid-fetch.
-    started_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True
-    )
     completed_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )

backend/app/models/import_settings.py

@@ -9,7 +9,6 @@ from sqlalchemy import (
     CheckConstraint,
     Float,
     Integer,
-    String,
     Text,
     select,
     text,
@@ -101,20 +100,6 @@ class ImportSettings(Base):
         Integer, nullable=False, default=3,
         server_default="3",
     )
-    # Off-site backup destination: an S3 bucket (services/backup_destination).
-    # The secret key is stored sealed with the credential key, never in clear.
-    backup_s3_endpoint: Mapped[str | None] = mapped_column(Text, nullable=True)
-    backup_s3_bucket: Mapped[str | None] = mapped_column(String(255), nullable=True)
-    backup_s3_access_key: Mapped[str | None] = mapped_column(Text, nullable=True)
-    backup_s3_secret_sealed: Mapped[str | None] = mapped_column(Text, nullable=True)
-    backup_s3_region: Mapped[str] = mapped_column(
-        String(64), nullable=False, default="us-east-1",
-        server_default="us-east-1",
-    )
-    # Upload bandwidth to that bucket, in megabits per second; 0 = no limit.
-    backup_upload_limit_mbps: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=200, server_default="200",
-    )
 
     # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
     # the weighted-score cut-off (0..1) above which a pending suggestion is made.

backend/app/services/backup_destination.py

@@ -1,138 +0,0 @@
-"""The off-site backup destination: an S3 bucket (Fabled Archives, or any S3
-service) that the database dumps and the media blobs are sent to.
-
-The address, bucket, access key and region are plain settings. The secret key
-is sealed at rest with the credential key (`/images/secrets`), the same key
-that seals account logins, and no API ever returns it: the settings answer
-only whether one is set. Because the credential key is kept out of every
-backup, a database dump carries the secret only as ciphertext nobody can open
-without the original server.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass
-from pathlib import Path
-from urllib.parse import urlsplit
-
-from .backup_s3 import S3Error, S3Target, list_objects
-from .credential_crypto import CredentialCrypto, InvalidCredentialBlob
-
-KEY_PATH = Path("/images/secrets/credential_key.b64")
-DEFAULT_REGION = "us-east-1"
-_TEST_LISTING = 5
-
-
-def _crypto() -> CredentialCrypto:
-    return CredentialCrypto(KEY_PATH)
-
-
-def seal(secret: str) -> str:
-    return _crypto().encrypt(secret).decode("ascii")
-
-
-def unseal(sealed: str) -> str:
-    return _crypto().decrypt(sealed.encode("ascii"))
-
-
-@dataclass(frozen=True)
-class DestinationFields:
-    endpoint: str | None
-    bucket: str | None
-    access_key: str | None
-    region: str
-
-
-def clean_fields(body: dict) -> tuple[DestinationFields | None, str | None]:
-    """The destination fields from a request body, trimmed, with empty ones as
-    None; or (None, why) when the address can't be an S3 endpoint."""
-    def text(name: str) -> str | None:
-        v = body.get(name)
-        if v is None:
-            return None
-        if not isinstance(v, str):
-            raise TypeError(name)
-        return v.strip() or None
-
-    try:
-        endpoint = text("endpoint")
-        bucket = text("bucket")
-        access_key = text("access_key")
-        region = text("region") or DEFAULT_REGION
-    except TypeError as exc:
-        return None, f"{exc.args[0]} must be text"
-    if endpoint is not None:
-        u = urlsplit(endpoint)
-        if u.scheme not in ("http", "https") or not u.hostname:
-            return None, ("the address must start with http:// or https:// and name "
-                          "a host, e.g. http://archives.example:8081")
-        if u.query or u.fragment:
-            return None, "the address can't carry a query or a fragment"
-        endpoint = endpoint.rstrip("/")
-    if bucket is not None and "/" in bucket:
-        return None, "the bucket is a name, not a path: put any path in the address"
-    return DestinationFields(endpoint, bucket, access_key, region), None
-
-
-def target_for(fields: DestinationFields, secret: str | None) -> S3Target | None:
-    """The S3 target, or None while any part of it is missing."""
-    if not (fields.endpoint and fields.bucket and fields.access_key and secret):
-        return None
-    return S3Target(fields.endpoint, fields.bucket, fields.access_key, secret,
-                    region=fields.region)
-
-
-def saved_target(settings) -> S3Target | None:
-    """The destination saved in ImportSettings, or None when none is set up.
-    Raises S3Error when the secret can't be opened (the credential key
-    changed since it was saved)."""
-    sealed = settings.backup_s3_secret_sealed
-    if not sealed:
-        return None
-    try:
-        secret = unseal(sealed)
-    except InvalidCredentialBlob:
-        raise S3Error(
-            "The saved secret key can't be opened on this server (the credential "
-            "key changed since it was saved). Enter the secret key again."
-        ) from None
-    return target_for(fields_of(settings), secret)
-
-
-def fields_of(settings) -> DestinationFields:
-    return DestinationFields(
-        settings.backup_s3_endpoint, settings.backup_s3_bucket,
-        settings.backup_s3_access_key, settings.backup_s3_region or DEFAULT_REGION,
-    )
-
-
-def describe(settings) -> dict:
-    """The destination as the settings screen shows it: never the secret."""
-    f = fields_of(settings)
-    return {
-        "endpoint": f.endpoint or "",
-        "bucket": f.bucket or "",
-        "access_key": f.access_key or "",
-        "region": f.region,
-        "secret_set": bool(settings.backup_s3_secret_sealed),
-        "upload_limit_mbps": settings.backup_upload_limit_mbps,
-        "configured": bool(
-            f.endpoint and f.bucket and f.access_key and settings.backup_s3_secret_sealed
-        ),
-    }
-
-
-def check_target(target: S3Target) -> dict:
-    """Reach the bucket by listing it. A listing proves the address, the
-    bucket and both keys at once, and writes nothing, because an Archives key
-    is allowed to add objects but a test object could never be removed."""
-    try:
-        page = list_objects(target, limit=_TEST_LISTING)
-    except S3Error as exc:
-        return {"ok": False, "message": str(exc)}
-    if not page.objects:
-        said = f"Reached {target.bucket} at {target.endpoint}. The bucket is empty."
-    else:
-        more = "+" if page.next_token else ""
-        said = (f"Reached {target.bucket} at {target.endpoint}. It holds "
-                f"{len(page.objects)}{more} object(s), e.g. {page.objects[0].key}.")
-    return {"ok": True, "message": said}

backend/app/services/backup_media.py

@@ -1,202 +0,0 @@
-"""The off-site copy of the library's files: every image and attachment, sent
-once each to the backup bucket as a content-addressed object.
-
-Layout in the bucket, beside the database dumps under `database/`:
-
-    media/ab/cd/<sha256><ext>
-
-`ab` and `cd` are the first four hex digits of the file's sha256, which keeps
-any one prefix small, and the extension keeps the files readable in the bucket
-without FabledCurator. The sha256 is the one recorded at import, so a file
-uploads exactly once however many rows or posts point at it, and a run is
-incremental: its work list is every sha with no `backup_blob` row for the
-current destination. A file removed from the library stays in the bucket until
-the bucket's own retention removes it; FabledCurator never deletes there.
-
-The bucket checks every object against that recorded sha256, so a file whose
-bytes changed on disk since import is refused and reported, never stored under
-the wrong name.
-"""
-from __future__ import annotations
-
-import re
-import time
-from dataclasses import dataclass
-from pathlib import Path
-
-from sqlalchemy import delete, select, text
-from sqlalchemy.dialects.postgresql import insert
-
-from ..models import BackupBlob
-from .backup_s3 import (
-    Pacer,
-    S3Error,
-    S3Target,
-    list_objects,
-    object_present,
-    put_file,
-    put_file_in_parts,
-)
-
-PREFIX = "media/"
-# Files above this size go in parts (S3 refuses a single PUT over 5 GiB, and a
-# part that fails is resent alone instead of the whole file).
-MULTIPART_OVER = 1 << 30
-# The bucket is listed this many keys at a time during a reconcile, with a
-# short rest between pages.
-LIST_PAGE = 1000
-LIST_REST_SECONDS = 0.2
-_EXT = re.compile(r"^\.[a-z0-9]{1,10}$")
-
-
-@dataclass(frozen=True)
-class Blob:
-    sha256: str
-    path: str
-    size_bytes: int
-    mime: str | None
-
-
-def destination_id(endpoint: str, bucket: str) -> str:
-    """How backup_blob names a destination: one copy per endpoint and bucket."""
-    return f"{endpoint.rstrip('/')}/{bucket}"
-
-
-def target_id(target: S3Target) -> str:
-    return destination_id(target.endpoint, target.bucket)
-
-
-def blob_key(sha256: str, path: str) -> str:
-    """The bucket key for a file: `media/ab/cd/<sha256><ext>`. An extension
-    that isn't short and plain is left off, so a key is always safe."""
-    ext = Path(path).suffix.lower()
-    return f"{PREFIX}{sha256[:2]}/{sha256[2:4]}/{sha256}{ext if _EXT.match(ext) else ''}"
-
-
-# Every library file once per sha256: images and attachments together. An
-# attachment's sha can repeat across posts, and an image extracted from an
-# archive is its own file with its own sha.
-_LIBRARY = """
-    SELECT sha256, path, size_bytes, mime FROM image_record
-    UNION ALL
-    SELECT sha256, path, size_bytes, mime FROM post_attachment
-"""
-
-
-def pending(session, tid: str, *, after: str = "", limit: int = 500) -> list[Blob]:
-    """The next `limit` files not yet in the bucket, in sha order after
-    `after`, so a run walks the library once even when some files fail."""
-    rows = session.execute(text(f"""
-        SELECT DISTINCT ON (lib.sha256) lib.sha256, lib.path, lib.size_bytes, lib.mime
-        FROM ({_LIBRARY}) AS lib
-        WHERE lib.sha256 > :after
-          AND NOT EXISTS (
-              SELECT 1 FROM backup_blob b WHERE b.target = :tid AND b.sha256 = lib.sha256
-          )
-        ORDER BY lib.sha256, lib.path
-        LIMIT :limit
-    """), {"after": after, "tid": tid, "limit": limit}).all()
-    return [Blob(r.sha256, r.path, r.size_bytes, r.mime) for r in rows]
-
-
-def coverage(session, tid: str | None) -> dict:
-    """How much of the library is in the bucket: files and bytes, sent and
-    total, each sha counted once."""
-    row = session.execute(text(f"""
-        WITH lib AS (
-            SELECT DISTINCT ON (sha256) sha256, size_bytes FROM ({_LIBRARY}) AS u
-            ORDER BY sha256
-        )
-        SELECT count(*) AS files_total,
-               coalesce(sum(lib.size_bytes), 0) AS bytes_total,
-               count(b.sha256) AS files_sent,
-               coalesce(sum(lib.size_bytes) FILTER (WHERE b.sha256 IS NOT NULL), 0)
-                   AS bytes_sent
-        FROM lib
-        LEFT JOIN backup_blob b ON b.sha256 = lib.sha256 AND b.target = :tid
-    """), {"tid": tid or ""}).one()
-    return {
-        "files_total": row.files_total, "bytes_total": int(row.bytes_total),
-        "files_sent": row.files_sent, "bytes_sent": int(row.bytes_sent),
-    }
-
-
-def send(target: S3Target, blob: Blob, images_root: Path, pacer: Pacer | None) -> str:
-    """Upload one file to its key and return the key. Raises S3Error (or
-    OSError) with the reason when it can't be sent."""
-    path = Path(blob.path)
-    if not path.is_absolute():
-        path = images_root / path
-    if not path.is_file():
-        raise S3Error(f"{path} is not on disk.")
-    size = path.stat().st_size
-    if size != blob.size_bytes:
-        raise S3Error(f"{path} is {size} bytes on disk but {blob.size_bytes} when imported.")
-    key = blob_key(blob.sha256, blob.path)
-    content_type = blob.mime or "application/octet-stream"
-    try:
-        if size > MULTIPART_OVER:
-            put_file_in_parts(target, key, path, sha256=blob.sha256,
-                              content_type=content_type, pacer=pacer)
-        else:
-            put_file(target, key, path, content_type=content_type,
-                     payload_sha256=blob.sha256, pacer=pacer)
-    except S3Error:
-        if object_present(target, key, size):
-            return key          # already there from an earlier run
-        raise
-    return key
-
-
-def record(session, tid: str, blobs: list[tuple[Blob, str]]) -> None:
-    """Note files as sent: (blob, key) pairs. A row already there is kept."""
-    if blobs:
-        session.execute(insert(BackupBlob).values([
-            {"target": tid, "sha256": b.sha256, "key": key, "size_bytes": b.size_bytes}
-            for b, key in blobs
-        ]).on_conflict_do_nothing())
-
-
-def reconcile(session, target: S3Target) -> dict:
-    """Check backup_blob against what the bucket actually holds under
-    `media/`. A row whose object is missing or the wrong size is dropped, so
-    the next run sends that file again; an object the bucket holds for a
-    library file with no row (a restored database, a lost commit) is recorded,
-    so it isn't sent twice. Returns the counts."""
-    tid = target_id(target)
-    held: dict[str, int] = {}
-    token = None
-    while True:
-        page = list_objects(target, PREFIX, LIST_PAGE, continuation=token)
-        held.update((o.key, o.size) for o in page.objects)
-        token = page.next_token
-        if not token:
-            break
-        time.sleep(LIST_REST_SECONDS)
-
-    rows = session.execute(
-        select(BackupBlob.sha256, BackupBlob.key, BackupBlob.size_bytes)
-        .where(BackupBlob.target == tid)
-    ).all()
-    gone = [r.sha256 for r in rows if held.get(r.key) != r.size_bytes]
-    for at in range(0, len(gone), 1000):
-        session.execute(delete(BackupBlob).where(
-            BackupBlob.target == tid, BackupBlob.sha256.in_(gone[at:at + 1000]),
-        ))
-    kept = {r.sha256 for r in rows} - set(gone)
-
-    found = []
-    for r in session.execute(text(f"""
-        SELECT DISTINCT ON (sha256) sha256, path, size_bytes, mime
-        FROM ({_LIBRARY}) AS lib ORDER BY sha256, path
-    """)).all():
-        if r.sha256 in kept:
-            continue
-        blob = Blob(r.sha256, r.path, r.size_bytes, r.mime)
-        key = blob_key(blob.sha256, blob.path)
-        if held.get(key) == blob.size_bytes:
-            found.append((blob, key))
-    for at in range(0, len(found), 1000):
-        record(session, tid, found[at:at + 1000])
-    session.commit()
-    return {"listed": len(held), "dropped": len(gone), "adopted": len(found)}

backend/app/services/backup_s3.py

@@ -1,505 +0,0 @@
-"""A small S3 client for sending backups off the box: AWS Signature Version 4,
-path-style addressing (endpoint/bucket/key), and the calls a backup needs —
-put an object (whole, or in parts when its size isn't known up front), read
-one back, list a prefix.
-
-FabledCurator puts, lists and reads; it never deletes. The bucket keeps the
-history and thins it on its own policy, so a key that may only add and read
-objects is enough, and is the safer key to hand a service: a stolen one cannot
-wipe the history it writes into.
-
-Built on the standard library's http.client rather than boto3: three calls
-don't justify a large dependency, and signing is a pure function the tests
-hold to AWS's own published examples. Every call is synchronous, because the
-uploads run in Celery workers; the API runs the Test destination call in a
-thread.
-"""
-from __future__ import annotations
-
-import hashlib
-import hmac
-import http.client
-import os
-import socket
-import ssl
-import time
-import xml.etree.ElementTree as ET
-from collections.abc import Iterator
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from pathlib import Path
-from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
-
-EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
-TIMEOUT_SECONDS = 30.0
-# A single object can be a multi-gigabyte video or archive; a slow link still
-# gets this long between bytes before the call gives up.
-TRANSFER_TIMEOUT_SECONDS = 15 * 60
-CHUNK = 1 << 20
-
-
-class S3Error(Exception):
-    """Why a call to the bucket failed, said to the person."""
-
-
-@dataclass(frozen=True)
-class S3Target:
-    endpoint: str          # http(s)://host[:port][/prefix], no bucket
-    bucket: str
-    access_key: str
-    secret_key: str
-    region: str = "us-east-1"
-    verify_tls: bool = True
-
-    def url(self, key: str = "") -> str:
-        base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}"
-        return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/"
-
-
-@dataclass(frozen=True)
-class StoredObject:
-    key: str
-    size: int
-    last_modified: str
-
-
-@dataclass(frozen=True)
-class ListPage:
-    objects: list[StoredObject]
-    next_token: str | None     # pass back as `continuation` for the next page
-
-
-# ── Signing ──────────────────────────────────────────────────────────────────
-
-def _encode(text: str, keep_slash: bool = False) -> str:
-    return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
-
-
-def _hmac(key: bytes, msg: str) -> bytes:
-    return hmac.new(key, msg.encode(), hashlib.sha256).digest()
-
-
-def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
-         access_key: str, secret_key: str, region: str, at: datetime,
-         service: str = "s3") -> dict[str, str]:
-    """The request's headers with Host, x-amz-date, x-amz-content-sha256 and
-    Authorization added. Every header given is signed, Host always."""
-    u = urlsplit(url)
-    stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
-    day = stamp[:8]
-    out = dict(headers)
-    out["Host"] = u.netloc
-    out["x-amz-date"] = stamp
-    out["x-amz-content-sha256"] = payload_sha256
-    signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
-    names = sorted(signed)
-
-    query = sorted(
-        (_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True)
-    )
-    canonical = "\n".join([
-        method.upper(),
-        _encode(unquote(u.path) or "/", keep_slash=True),
-        "&".join(f"{k}={v}" for k, v in query),
-        "".join(f"{n}:{signed[n]}\n" for n in names),
-        ";".join(names),
-        payload_sha256,
-    ])
-    scope = f"{day}/{region}/{service}/aws4_request"
-    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
-                         hashlib.sha256(canonical.encode()).hexdigest()])
-    key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
-                      service), "aws4_request")
-    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
-    out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
-                            f"SignedHeaders={';'.join(names)}, Signature={signature}")
-    return out
-
-
-# ── Transport ────────────────────────────────────────────────────────────────
-
-def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection:
-    u = urlsplit(url)
-    if u.scheme == "https":
-        ctx = ssl.create_default_context()
-        if not target.verify_tls:
-            ctx.check_hostname = False
-            ctx.verify_mode = ssl.CERT_NONE
-        return http.client.HTTPSConnection(
-            u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK,
-        )
-    if u.scheme != "http":
-        raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.")
-    return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK)
-
-
-def _request_target(url: str) -> str:
-    u = urlsplit(url)
-    return (u.path or "/") + (f"?{u.query}" if u.query else "")
-
-
-def _reach_error(target: S3Target, exc: Exception) -> S3Error:
-    if isinstance(exc, TimeoutError):
-        return S3Error(f"{target.endpoint} didn't answer in time.")
-    if isinstance(exc, socket.gaierror):
-        return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.")
-    if isinstance(exc, ConnectionRefusedError):
-        return S3Error(
-            f"Couldn't reach {target.endpoint}: nothing is listening there "
-            "(check the port)."
-        )
-    if isinstance(exc, ssl.SSLError):
-        return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).")
-    return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
-
-
-def _says(status: int, reason: str, content: bytes) -> str:
-    """An S3 error response in words: its Code and Message when it has them."""
-    try:
-        root = ET.fromstring(content) if content else None
-        code = (root.findtext("Code") or "") if root is not None else ""
-        message = (root.findtext("Message") or "") if root is not None else ""
-    except ET.ParseError:
-        code, message = "", ""
-    hints = {
-        "SignatureDoesNotMatch": "the secret key is wrong",
-        "InvalidAccessKeyId": "the access key isn't known there",
-        "NoSuchBucket": "there is no such bucket",
-        "AccessDenied": "the key isn't allowed to do that",
-        "XAmzContentSHA256Mismatch": "the file's bytes don't match its recorded checksum",
-    }
-    said = hints.get(code) or message or reason
-    if not code and 400 <= status < 500:
-        # S3 refuses with an <Error><Code> body. A 4xx without one came from
-        # something else at that address — most often a web server on the
-        # default port, when the S3 service listens on a port of its own.
-        said = ("the address answered, but not as S3 — check the scheme and "
-                "the port (an S3 service usually listens on its own port)")
-    return f"HTTP {status}{f' {code}' if code else ''}: {said}."
-
-
-# ── Calls ────────────────────────────────────────────────────────────────────
-
-def file_sha256(path: Path) -> str:
-    h = hashlib.sha256()
-    with open(path, "rb") as f:
-        while chunk := f.read(CHUNK):
-            h.update(chunk)
-    return h.hexdigest()
-
-
-def put_file(target: S3Target, key: str, path: Path, *,
-             content_type: str = "application/octet-stream",
-             payload_sha256: str | None = None,
-             pacer: Pacer | None = None,
-             now: datetime | None = None) -> None:
-    """Upload one file as one object, streamed from disk. `payload_sha256` is
-    the file's sha256 when the caller already knows it (the library stores
-    every file's), which saves reading the file twice; the bucket checks the
-    bytes against it, so a file that changed on disk is refused rather than
-    stored under the wrong name."""
-    digest = payload_sha256 or file_sha256(path)
-    size = path.stat().st_size
-    url = target.url(key)
-    headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type},
-                   digest, target.access_key, target.secret_key, target.region,
-                   now or datetime.now(UTC))
-    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
-    try:
-        with open(path, "rb") as f:
-            conn.request("PUT", _request_target(url),
-                         body=pacer.file_chunks(f) if pacer else f, headers=headers)
-            response = conn.getresponse()
-            content = response.read()
-    except (OSError, http.client.HTTPException) as exc:
-        raise _reach_error(target, exc) from None
-    finally:
-        conn.close()
-    if response.status // 100 != 2:
-        raise S3Error(f"Upload of {key} refused: "
-                      f"{_says(response.status, response.reason, content)}")
-
-
-def get_file(target: S3Target, key: str, path: Path, *,
-             now: datetime | None = None) -> None:
-    """Download one object to `path`, streamed to disk. The bytes land under a
-    `.partial` name and are renamed into place when complete, so a broken
-    transfer never leaves a truncated file at `path`."""
-    url = target.url(key)
-    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
-                   target.region, now or datetime.now(UTC))
-    partial = path.with_name(path.name + ".partial")
-    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
-    try:
-        conn.request("GET", _request_target(url), headers=headers)
-        response = conn.getresponse()
-        if response.status != 200:
-            raise S3Error(f"Reading {key} refused: "
-                          f"{_says(response.status, response.reason, response.read())}")
-        with open(partial, "wb") as f:
-            while chunk := response.read(CHUNK):
-                f.write(chunk)
-        os.replace(partial, path)
-    except (OSError, http.client.HTTPException) as exc:
-        partial.unlink(missing_ok=True)
-        raise _reach_error(target, exc) from None
-    finally:
-        conn.close()
-
-
-def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
-                 continuation: str | None = None,
-                 now: datetime | None = None) -> ListPage:
-    """One page (up to `limit`) of the objects under `prefix`, ListObjectsV2."""
-    params = {"list-type": "2", "max-keys": str(limit)}
-    if prefix:
-        params["prefix"] = prefix
-    if continuation:
-        params["continuation-token"] = continuation
-    url = f"{target.url()}?{urlencode(params, quote_via=quote)}"
-    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
-                   target.region, now or datetime.now(UTC))
-    conn = _connect(target, url, TIMEOUT_SECONDS)
-    try:
-        conn.request("GET", _request_target(url), headers=headers)
-        response = conn.getresponse()
-        content = response.read()
-    except (OSError, http.client.HTTPException) as exc:
-        raise _reach_error(target, exc) from None
-    finally:
-        conn.close()
-    if response.status != 200:
-        raise S3Error(f"Listing {target.bucket} refused: "
-                      f"{_says(response.status, response.reason, content)}")
-    try:
-        root = ET.fromstring(content)
-    except ET.ParseError:
-        raise S3Error(
-            f"{target.endpoint} answered a listing with something that isn't S3's XML."
-        ) from None
-    objects, next_token = [], None
-    for item in root.iter():
-        tag = item.tag.rsplit("}", 1)[-1]
-        if tag == "NextContinuationToken":
-            next_token = item.text or None
-        if tag != "Contents":
-            continue
-        fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
-        objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
-                                    fields.get("LastModified", "")))
-    return ListPage(objects, next_token)
-
-
-# ── Multipart: one object sent in parts ─────────────────────────────────────
-
-# Each part is held in memory while it is sent, so this bounds the memory a
-# streamed upload uses. S3 allows 10,000 parts, which at this size is 640 GiB
-# for one object, and every part but the last must be at least 5 MiB.
-PART_SIZE = 64 * CHUNK
-# A part the bucket couldn't take (it didn't answer, or said it was busy) is
-# sent again after a rest that grows each time: 15 s, then 60 s.
-PART_RETRY_WAITS = (15, 60)
-_BUSY = frozenset({429, 500, 502, 503, 504})
-_MAX_RETRY_AFTER = 300
-
-
-class Pacer:
-    """Holds the bytes a job sends to an average rate, across every request
-    it makes, so a backup leaves room on the link for everything else."""
-
-    def __init__(self, mbps: int | None):
-        self.rate = mbps * 1_000_000 / 8 if mbps else None   # bytes per second
-        self.sent = 0
-        self.started = time.monotonic()
-
-    def _wait(self, n: int) -> None:
-        if self.rate:
-            self.sent += n
-            ahead = self.sent / self.rate - (time.monotonic() - self.started)
-            if ahead > 0:
-                time.sleep(ahead)
-
-    def chunks(self, data: bytes) -> Iterator[memoryview]:
-        view = memoryview(data)
-        for at in range(0, len(view), CHUNK):
-            piece = view[at:at + CHUNK]
-            self._wait(len(piece))
-            yield piece
-
-    def file_chunks(self, f) -> Iterator[bytes]:
-        while piece := f.read(CHUNK):
-            self._wait(len(piece))
-            yield piece
-
-
-def _send(target: S3Target, method: str, url: str, body: bytes = b"",
-          headers: dict[str, str] | None = None, *, timeout: float = TIMEOUT_SECONDS,
-          pacer: Pacer | None = None,
-          now: datetime | None = None) -> tuple[int, str, http.client.HTTPMessage, bytes]:
-    """Sign and send one request with an in-memory body, paced when a pacer
-    is given. Returns (status, reason, headers, content)."""
-    digest = hashlib.sha256(body).hexdigest()
-    signed = sign(method, url, {"Content-Length": str(len(body)), **(headers or {})},
-                  digest, target.access_key, target.secret_key, target.region,
-                  now or datetime.now(UTC))
-    conn = _connect(target, url, timeout)
-    try:
-        conn.request(method, _request_target(url),
-                     body=pacer.chunks(body) if pacer and body else body, headers=signed)
-        response = conn.getresponse()
-        content = response.read()
-    except (OSError, http.client.HTTPException) as exc:
-        raise _reach_error(target, exc) from None
-    finally:
-        conn.close()
-    return response.status, response.reason, response.headers, content
-
-
-def _xml_field(content: bytes, name: str) -> str | None:
-    try:
-        root = ET.fromstring(content)
-    except ET.ParseError:
-        return None
-    for item in root.iter():
-        if item.tag.rsplit("}", 1)[-1] == name:
-            return item.text
-    return None
-
-
-def _is_error_body(content: bytes) -> bool:
-    try:
-        return bool(content) and ET.fromstring(content).tag.rsplit("}", 1)[-1] == "Error"
-    except ET.ParseError:
-        return False
-
-
-def _retry_after(headers: http.client.HTTPMessage | None, default: int) -> int:
-    try:
-        asked = int((headers or {}).get("Retry-After") or default)
-    except ValueError:
-        asked = default
-    return max(0, min(asked, _MAX_RETRY_AFTER))
-
-
-class Upload:
-    """One object written as a series of parts, for data whose size isn't known
-    when it starts, such as a database dump streamed from pg_dump.
-
-    The object appears in the bucket only when `complete()` succeeds. Until
-    then the parts are invisible, so a stream that fails partway leaves
-    nothing behind that looks like a finished backup. `abort()` asks the
-    bucket to drop the parts at once; if the abort is refused or never
-    arrives, S3 services expire an unfinished upload on their own (Fabled
-    Archives after a day)."""
-
-    def __init__(self, target: S3Target, key: str, *,
-                 content_type: str = "application/octet-stream",
-                 pacer: Pacer | None = None):
-        self.target = target
-        self.key = key
-        self.size = 0
-        self.pacer = pacer
-        self._etags: list[str] = []
-        status, reason, _, content = _send(
-            target, "POST", f"{target.url(key)}?uploads", headers={"Content-Type": content_type},
-        )
-        upload_id = _xml_field(content, "UploadId") if status == 200 else None
-        if not upload_id:
-            raise S3Error(f"Starting the upload of {key} refused: "
-                          f"{_says(status, reason, content)}")
-        self.upload_id = upload_id
-
-    def _url(self, **params: str) -> str:
-        query = urlencode({**params, "uploadId": self.upload_id}, quote_via=quote)
-        return f"{self.target.url(self.key)}?{query}"
-
-    @property
-    def parts(self) -> int:
-        return len(self._etags)
-
-    def add(self, data: bytes) -> None:
-        """Send the next part, resting and sending it again when the bucket
-        doesn't answer or says it is busy."""
-        n = len(self._etags) + 1
-        for wait in (*PART_RETRY_WAITS, None):
-            headers = None
-            try:
-                status, reason, headers, content = _send(
-                    self.target, "PUT", self._url(partNumber=str(n)), data,
-                    timeout=TRANSFER_TIMEOUT_SECONDS, pacer=self.pacer,
-                )
-            except S3Error as exc:
-                if wait is None:
-                    raise S3Error(f"Part {n} of {self.key}: {exc}") from None
-                time.sleep(wait)
-                continue
-            etag = headers.get("ETag") if status == 200 else None
-            if etag:
-                self._etags.append(etag)
-                self.size += len(data)
-                return
-            if status not in _BUSY or wait is None:
-                raise S3Error(f"Part {n} of {self.key} refused: "
-                              f"{_says(status, reason, content)}")
-            time.sleep(_retry_after(headers, wait))
-
-    def complete(self) -> None:
-        """Join the parts into the object. S3 can refuse a completion with an
-        error body under a 200 status, so the body is read too."""
-        if not self._etags:
-            raise S3Error(f"Nothing was sent for {self.key}.")
-        body = "<CompleteMultipartUpload>" + "".join(
-            f"<Part><PartNumber>{n}</PartNumber><ETag>{etag}</ETag></Part>"
-            for n, etag in enumerate(self._etags, start=1)
-        ) + "</CompleteMultipartUpload>"
-        status, reason, _, content = _send(
-            self.target, "POST", self._url(), body.encode(),
-            headers={"Content-Type": "application/xml"}, timeout=TRANSFER_TIMEOUT_SECONDS,
-        )
-        if status != 200 or _is_error_body(content):
-            raise S3Error(f"Finishing the upload of {self.key} refused: "
-                          f"{_says(status, reason, content)}")
-
-    def abort(self) -> None:
-        """Drop the parts sent so far. Never raises: it runs while another
-        failure is already being reported, and the bucket expires the parts on
-        its own if this doesn't reach it."""
-        try:
-            _send(self.target, "DELETE", self._url())
-        except Exception:  # noqa: BLE001 — best effort, see above
-            pass
-
-
-def put_file_in_parts(target: S3Target, key: str, path: Path, *, sha256: str,
-                      content_type: str = "application/octet-stream",
-                      pacer: Pacer | None = None) -> int:
-    """Upload a large file as a multipart object, read from disk a part at a
-    time. The whole file's sha256 is computed on the way and must equal
-    `sha256` before the upload is completed; otherwise it is aborted, so a file
-    that changed on disk is never stored under its old name. Returns the
-    number of bytes sent."""
-    upload = Upload(target, key, content_type=content_type, pacer=pacer)
-    digest = hashlib.sha256()
-    try:
-        with open(path, "rb") as f:
-            while part := f.read(PART_SIZE):
-                digest.update(part)
-                upload.add(part)
-        if digest.hexdigest() != sha256:
-            raise S3Error(f"{path} no longer matches its recorded checksum; not stored.")
-        upload.complete()
-    except BaseException:
-        upload.abort()
-        raise
-    return upload.size
-
-
-def object_present(target: S3Target, key: str, size: int) -> bool:
-    """Whether `key` is already in the bucket at `size` bytes. Used when a put
-    is refused: an append-only bucket refuses to replace an object that is
-    already there, which for a content-addressed key means the work is done."""
-    try:
-        page = list_objects(target, prefix=key, limit=1)
-    except S3Error:
-        return False
-    return any(o.key == key and o.size == size for o in page.objects)

backend/app/services/backup_service.py

@@ -4,9 +4,7 @@ Two independent backup kinds:
   - 'db'     — pg_dump only; fast; nightly via Beat (settings-gated)
   - 'images' — tar+zstd of /images; slow; manual trigger only
 
-With an off-site destination set up, a database dump streams straight to that
-bucket instead (backup_db_offsite). Otherwise files live under
-<images_root>/_backups/. Each local backup writes:
+Files live under <images_root>/_backups/. Each backup writes:
   fc_<kind>_<ts>.{sql|tar.zst}        — the artifact
   fc_<kind>_<ts>.json                 — manifest (kind/tag/triggered_by)
 
@@ -21,7 +19,6 @@ import os
 import shutil
 import subprocess
 import tempfile
-import time
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -156,10 +153,10 @@ def backup_db(
     tmp_path = Path(tmp_name)
     try:
         _run_bounded(
-            _gently([
+            [
                 "pg_dump", "--no-owner", "--no-acl", "-Fc",
                 "-f", str(tmp_path), _libpq_url(db_url),
-            ]),
+            ],
             _DB_SUBPROCESS_TIMEOUT_S,
         )
         shutil.move(str(tmp_path), str(sql_path))
@@ -180,79 +177,6 @@ def backup_db(
     }
 
 
-class DumpFailed(Exception):
-    """pg_dump exited with an error; the message is its last line of output."""
-
-
-def offsite_dump_key(ts: str) -> str:
-    """Where a database dump lands in the off-site bucket. Dated names sit
-    side by side under one prefix, which is what lets the bucket's retention
-    thin them by age."""
-    return f"database/fabledcurator-{ts}.dump"
-
-
-def _gently(cmd: list[str]) -> list[str]:
-    """`cmd` at low CPU priority, and low disk priority where `ionice` is
-    installed, so a long dump leaves the server responsive."""
-    io = ["ionice", "-c", "2", "-n", "7"] if shutil.which("ionice") else []
-    return ["nice", "-n", "10", *io, *cmd]
-
-
-def _last_line(f) -> str | None:
-    f.seek(0)
-    lines = f.read().decode(errors="replace").strip().splitlines()
-    return lines[-1].strip() if lines else None
-
-
-def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) -> dict:
-    """Stream `pg_dump` straight into the off-site bucket, in parts, with no
-    copy staged on this server. The object appears only once pg_dump has
-    exited cleanly and every part has arrived; a dump that fails partway
-    leaves nothing in the bucket, and raises DumpFailed with pg_dump's own
-    last line. Returns a dict for the caller to record in BackupRun."""
-    from .backup_s3 import PART_SIZE, Pacer, Upload
-
-    ts = _now_ts()
-    key = offsite_dump_key(ts)
-    deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
-    with tempfile.TemporaryFile() as err:
-        proc = subprocess.Popen(
-            _gently(["pg_dump", "--no-owner", "--no-acl", "-Fc", _libpq_url(db_url)]),
-            stdout=subprocess.PIPE, stderr=err,
-        )
-        upload = None
-        try:
-            upload = Upload(target, key, pacer=Pacer(limit_mbps))
-            while part := proc.stdout.read(PART_SIZE):
-                if time.monotonic() > deadline:
-                    raise subprocess.TimeoutExpired(proc.args, _DB_SUBPROCESS_TIMEOUT_S)
-                upload.add(part)
-            proc.stdout.close()
-            proc.wait(timeout=_KILL_REAP_GRACE_S * 6)
-            if proc.returncode != 0:
-                raise DumpFailed(_last_line(err) or f"pg_dump exited with {proc.returncode}")
-            upload.complete()
-        except BaseException:
-            if proc.poll() is None:
-                proc.kill()
-                try:
-                    proc.wait(timeout=_KILL_REAP_GRACE_S)
-                except subprocess.TimeoutExpired:
-                    pass  # unkillable (D-state): abandon the reap, fail fast
-            if upload is not None:
-                upload.abort()
-            raise
-    return {
-        "kind": "db",
-        "ts": ts,
-        "key": key,
-        "bucket": target.bucket,
-        "endpoint": target.endpoint,
-        "parts": upload.parts,
-        "size_bytes": upload.size,
-    }
-
-
 def backup_images(
     *, images_root: Path,
     tag: str | None = None, triggered_by: str = "manual",

backend/app/services/download_service.py

@@ -369,8 +369,11 @@ class DownloadService:
                     if do_embed:
                         embed_image.delay(img_id)
             elif result.status == "attached":
-                # Non-media, or an archive with no new images, captured as a
-                # PostAttachment; the attachments store holds the copy.
+                # Non-media or extracted archive captured as PostAttachment
+                # (FC-2d-iii). The canonical copy lives in the attachments
+                # store; the original download path is now redundant —
+                # mirror duplicate_hash cleanup so we don't keep two copies.
+                # Operator-flagged 2026-06-02 (Lustria OST zip).
                 import_summary["attached"] += 1
                 # An archive captured WITHOUT extracting any image carries a
                 # reason on result.error — record it so the event explains the
@@ -387,13 +390,35 @@ class DownloadService:
                     bytes_downloaded += path.stat().st_size  # noqa: ASYNC240
                 except OSError:
                     pass
+                try:
+                    path.unlink(missing_ok=True)  # noqa: ASYNC240
+                except OSError:
+                    pass
+            elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
+                "duplicate_hash", "duplicate_phash",
+            ):
+                import_summary["skipped"] += 1
+                try:
+                    path.unlink(missing_ok=True)  # noqa: ASYNC240
+                except OSError:
+                    pass
             elif result.status == "skipped":
-                # A duplicate, or a file the import filters rejected. Neither
-                # is a download failure, so the run is not flagged.
+                # Soft skip (too_small, too_transparent, invalid_image) —
+                # the file just didn't qualify, not a download/ingest
+                # failure. Don't flag the run as error; the file stays
+                # on disk for operator inspection.
                 import_summary["skipped"] += 1
             elif result.status == "failed":
-                # Hard failure: an archive probe crashed or timed out.
+                # Hard failure (today only: archive probe crash/timeout).
+                # The original archive sits in /images/ as an orphan; the
+                # filesystem scanner would re-import and re-crash on the
+                # same file, so delete the source file and surface the
+                # error in import_summary. Audit 2026-06-02.
                 import_summary["errors"] += 1
+                try:
+                    path.unlink(missing_ok=True)  # noqa: ASYNC240
+                except OSError:
+                    pass
             elif result.status == "refreshed":
                 # Currently unreachable from attach_in_place (the download
                 # path never runs in deep=True mode), but the importer's
@@ -402,10 +427,6 @@ class DownloadService:
                 import_summary["attached"] += 1
             else:
                 import_summary["errors"] += 1
-            # Remove the download when the library no longer needs it at this
-            # path: an attachment or archive now in the store, a duplicate, a
-            # rejected file. One policy for every download path.
-            self.importer.discard_after_attach(path, result)
 
         # Post-only records: media-less posts (pure-text) the native ingester
         # captured so the artist archive is complete. Upsert each (keyed on

backend/app/services/external_fetch.py

@@ -241,19 +241,11 @@ def _fetch_gdrive(url: str, dest_dir: Path, *, read_timeout: float,
 def _fetch_mega(url: str, dest_dir: Path, *, read_timeout: float,
                 total_timeout: float,
                 should_stop: Callable[[], bool]) -> FetchResult:
-    # A folder link keeps its subfolders, so files are found at any depth.
-    # Dotfiles are megatools' `.megatmp.*` staging names.
-    def _complete_files() -> set[Path]:
-        return {
-            p for p in dest_dir.rglob("*")
-            if p.is_file() and not p.name.startswith(".")
-        } if dest_dir.exists() else set()
-
-    before = _complete_files()
+    before = set(dest_dir.iterdir()) if dest_dir.exists() else set()
     # megatools is a subprocess: its timeout IS a total wall-clock cap (the read
     # timeout has no analogue here), so the total budget applies directly.
     _run_mega_get(url, str(dest_dir), timeout=total_timeout)
-    new = sorted(_complete_files() - before)
+    new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()]
     if not new:
         return FetchResult(error="mega-get wrote no new file")
     return FetchResult(files=new, bytes=sum(p.stat().st_size for p in new))

backend/app/services/external_links.py

@@ -21,9 +21,9 @@ log = logging.getLogger(__name__)
 
 
 def refetch_links_for_post(session: Session, post_id: int) -> dict:
-    """Reset every settled ExternalLink on a post to pending (fresh attempt and
-    recovery budgets) and dispatch its fetch. In-flight ('downloading') links
-    are left alone. Commits. Returns {links_total, links_reset}."""
+    """Reset every settled ExternalLink on a post to pending (fresh attempt
+    budget) and dispatch its fetch. In-flight ('downloading') links are left
+    alone. Commits. Returns {links_total, links_reset}."""
     links = session.execute(
         select(ExternalLink).where(ExternalLink.post_id == post_id)
     ).scalars().all()
@@ -33,7 +33,6 @@ def refetch_links_for_post(session: Session, post_id: int) -> dict:
             continue
         link.status = "pending"
         link.attempts = 0
-        link.recoveries = 0
         link.last_error = None
         link.completed_at = None
         reset_ids.append(link.id)

backend/app/services/importer.py

@@ -16,7 +16,6 @@ from dataclasses import dataclass, field
 from enum import StrEnum
 from pathlib import Path
 
-from celery.exceptions import SoftTimeLimitExceeded
 from PIL import Image
 from sqlalchemy import func, select, update
 from sqlalchemy.exc import IntegrityError
@@ -44,7 +43,6 @@ from ..utils.paths import (
 from ..utils.phash import compute_phash, find_similar, fingerprint_path, fingerprints_match
 from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
-from ..utils.staging import is_temp_staging_file
 from .archive_extractor import extract_archive, is_archive
 from .attachment_store import AttachmentStore
 from .audits import single_color
@@ -117,21 +115,6 @@ def is_video(path: Path) -> bool:
     return path.suffix.lower() in VIDEO_EXTS
 
 
-def _not_content(path: Path) -> ImportResult | None:
-    """The skip result for a file that is never library content: a sidecar
-    `.json` (metadata) or a downloader's staging file (an unfinished download,
-    which becomes content only under its final name). None for anything else."""
-    if path.suffix.lower() == ".json":
-        error = "sidecar json is metadata, not content"
-    elif is_temp_staging_file(path):
-        error = "unfinished download (staging file), not content"
-    else:
-        return None
-    return ImportResult(
-        status="skipped", skip_reason=SkipReason.invalid_image, error=error,
-    )
-
-
 def _safe_ext(path: Path) -> str:
     """Conservatively extract a file extension for PostAttachment.ext
     (varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for
@@ -308,44 +291,6 @@ class Importer:
             sp.rollback()
             return self.session.execute(stmt).scalar_one()
 
-    def _store_new_record(self, record: ImageRecord) -> ImageRecord | None:
-        """Insert `record`, or return the row that already holds its bytes.
-
-        The sha256 lookup before an insert can lose a race: two copies of one
-        picture in different folders, imported side by side by two workers,
-        both find no row, and the unique index refuses the second insert. Only
-        that insert is undone (a savepoint), so the session stays usable and
-        the caller files its file as a duplicate of the winner. None means
-        `record` was stored."""
-        sp = self.session.begin_nested()
-        try:
-            self.session.add(record)
-            self.session.flush()
-            sp.commit()
-            return None
-        except IntegrityError:
-            sp.rollback()
-            winner = self.session.execute(
-                select(ImageRecord).where(ImageRecord.sha256 == record.sha256)
-            ).scalar_one_or_none()
-            if winner is None:
-                raise
-            return winner
-
-    def _file_as_duplicate(
-        self, existing: ImageRecord, path: Path, artist: Artist | None,
-        explicit_source: Source | None,
-    ) -> ImportResult:
-        """A byte-identical copy of `existing`: link the copy's post to the
-        image (enrich-on-duplicate), so a picture posted twice shows on both
-        posts, and report the skip."""
-        self._apply_sidecar(existing, path, artist, explicit_source=explicit_source)
-        self.session.commit()
-        return ImportResult(
-            status="skipped", skip_reason=SkipReason.duplicate_hash,
-            image_id=existing.id, error="sha256 already present",
-        )
-
     def _find_or_create_source(
         self, *, artist_id: int, platform: str, url: str,
     ) -> Source:
@@ -465,8 +410,11 @@ class Importer:
         media members (one Post via the archive-adjacent sidecar) and
         preserve the archive itself. Other non-media → PostAttachment.
         Nothing is skipped for being non-media (FC-2d-iii)."""
-        if (refused := _not_content(source)) is not None:
-            return refused
+        if source.suffix.lower() == ".json":
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.invalid_image,
+                error="sidecar json is metadata, not content",
+            )
         if is_archive(source):
             return self._import_archive(source)
         if not is_supported(source):
@@ -616,9 +564,6 @@ class Importer:
 
         artist_use = artist if artist is not None else self._resolve_artist(source)
         post = self._post_for_sidecar(source, artist_use, source_row=source_row)
-        # Committed before extraction, so a member that fails (and is rolled
-        # back on its own) can never take the archive's post with it.
-        self.session.commit()
         member_ids: list[int] = []
         # Every member image touched (new + superseded + deduped), so the
         # from_attachment_id stamp below covers files that already existed in the
@@ -745,15 +690,7 @@ class Importer:
                             member_record_ids.add(res.image_id)
                     else:
                         counts["failed"] += 1
-        except SoftTimeLimitExceeded:
-            # The task's time budget is spent: stop here so the task marks
-            # its row failed, rather than carrying on until the hard kill.
-            raise
         except Exception as exc:  # noqa: BLE001 — defensive per level; keep going
-            # Each member commits on its own, so this undoes only the one in
-            # progress; without it the next query fails on the dead transaction
-            # and the archive is never stored.
-            self.session.rollback()
             log.warning(
                 "archive extraction failed for %s (depth %d): %s",
                 archive_path.name, depth, exc,
@@ -934,8 +871,18 @@ class Importer:
         existing = self.session.execute(existing_stmt).scalar_one_or_none()
         if existing:
             if not self.deep:
-                return self._file_as_duplicate(
-                    existing, attribution_path, path_artist, explicit_source,
+                # Enrich-on-duplicate (parity with attach_in_place): a re-scanned
+                # file that is a byte-dup of an existing image still links its
+                # post via _apply_sidecar, so a cross-posted image shows on every
+                # post.
+                self._apply_sidecar(
+                    existing, attribution_path, path_artist,
+                    explicit_source=explicit_source,
+                )
+                self.session.commit()
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.duplicate_hash,
+                    image_id=existing.id, error="sha256 already present",
                 )
             return self._deep_rederive(existing, source, attribution_path)
 
@@ -1021,14 +968,8 @@ class Importer:
             origin="imported_filesystem",
             integrity_status="unknown",
         )
-        if (winner := self._store_new_record(record)) is not None:
-            # The copy just written is a stray unless it is the winner's own
-            # file (the same source imported twice lands on the same path).
-            if Path(winner.path) != dest:
-                dest.unlink(missing_ok=True)
-            return self._file_as_duplicate(
-                winner, attribution_path, path_artist, explicit_source,
-            )
+        self.session.add(record)
+        self.session.flush()
         self._phash_cache_append(phash, width, height, record.id)
 
         # Folder→artist (path_artist resolved up-front above); bind it to the
@@ -1278,8 +1219,11 @@ class Importer:
             caller may delete the on-disk original (mirrors duplicate_hash)
           - failed → file untouched; caller decides
         """
-        if (refused := _not_content(path)) is not None:
-            return refused
+        if path.suffix.lower() == ".json":
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.invalid_image,
+                error="sidecar json is metadata, not content",
+            )
         if is_archive(path):
             return self._import_archive(
                 path, artist=artist, source_row=source,
@@ -1356,7 +1300,12 @@ class Importer:
             # only sets it when NULL. Without this the new post is left with no
             # image AND (if its other content also deduped) no attachment, i.e. a
             # bare shell — operator-flagged 2026-06-08 (1589 empty Anduo posts).
-            return self._file_as_duplicate(existing, path, artist, source)
+            self._apply_sidecar(existing, path, artist, explicit_source=source)
+            self.session.commit()
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.duplicate_hash,
+                image_id=existing.id, error="sha256 already present",
+            )
 
         # phash dedup + supersede
         phash = None
@@ -1436,8 +1385,8 @@ class Importer:
         )
         if artist is not None:
             record.artist_id = artist.id
-        if (winner := self._store_new_record(record)) is not None:
-            return self._file_as_duplicate(winner, path, artist, source)
+        self.session.add(record)
+        self.session.flush()
         self._phash_cache_append(phash, width, height, record.id)
 
         # Sidecar provenance (best-effort). When `source` is passed, link
@@ -1452,51 +1401,6 @@ class Importer:
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
-    def discard_after_attach(self, path: Path, result: ImportResult) -> bool:
-        """Delete a downloaded file that `attach_in_place` has finished with,
-        when keeping it would only leave a second copy or a rejected file on
-        disk. Returns True when the file was removed. Both download paths (a
-        subscription walk's import step and the external-link worker) apply
-        this one policy:
-
-        - attached: the attachments store holds the canonical copy.
-        - imported or superseded ARCHIVE: its new images are in the library and
-          the archive itself is in the attachments store.
-        - duplicate skip: the library already holds the content.
-        - soft skip (too small, too transparent, single colour, not a readable
-          image): the import filters rejected it. A file left on disk with no
-          record is picked up by the next walk and imported again; the seen
-          ledger is what stops it being downloaded again.
-        - failed: the archive probe crashed, and a retry would crash again.
-
-        An imported or superseded media file is kept: it IS the record's file.
-        So is any path an ImageRecord points at, whatever the status, because a
-        re-fetch can write identical bytes over a record's own file. A staging
-        file is kept too: it may be a download still being written, and the
-        temp-file sweep removes it once it is old enough to be abandoned."""
-        if is_temp_staging_file(path):
-            return False
-        if result.status in ("imported", "superseded"):
-            redundant = is_archive(path)
-        else:
-            redundant = result.status in ("attached", "skipped", "failed")
-        if not redundant:
-            return False
-        owned = self.session.execute(
-            select(ImageRecord.id).where(
-                ImageRecord.path.in_({str(path), str(path.resolve())})
-            ).limit(1)
-        ).first()
-        if owned is not None:
-            log.info("keeping %s: it is image %s's file", path.name, owned.id)
-            return False
-        try:
-            path.unlink(missing_ok=True)
-        except OSError as exc:
-            log.warning("could not remove %s: %s", path, exc)
-            return False
-        return True
-
     def _upsert_artist(self, name: str) -> Artist:
         slug = slugify(name)
         artist = self.session.execute(

backend/app/services/library_scan.py

@@ -1,181 +0,0 @@
-"""Files in the artists' folders that the library has no record of.
-
-A download writes its files into the artist's folder and imports them in a
-later step. A run that stops between the two (a restart, a time limit) leaves
-complete files on disk that no image or attachment points at, and a walk that
-has already marked them seen never comes back for them. `find_unimported`
-lists those files; `queue_unimported` turns the same list into an import batch
-that attaches each file where it lies (`scan_mode="library"`), so a file the
-library already holds, or one the import filters reject, is removed instead.
-
-Only files under a folder named for an existing artist (matched by slug, so a
-differently-cased folder still matches) are considered, and only images,
-videos and archives. Files changed in the last day are left for the download
-that may still be writing them.
-"""
-from __future__ import annotations
-
-import os
-import stat
-import time
-from collections import defaultdict
-from dataclasses import dataclass
-from pathlib import Path
-
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from ..models import Artist, ImageRecord, ImportBatch, ImportTask, PostAttachment, Source
-from ..utils.slug import slugify
-from ..utils.staging import is_temp_staging_file
-from .archive_extractor import is_archive
-from .importer import is_supported
-
-# Top-level folders that hold the library's own stores, not an artist's files.
-_STORE_DIRS = frozenset({"thumbs", "attachments", "cookies", "secrets"})
-MIN_AGE_SECONDS = 24 * 3600
-_ACTIVE_TASK_STATUSES = ("pending", "queued", "processing")
-_SAMPLES_PER_ARTIST = 3
-_ARTISTS_LISTED = 50
-
-
-@dataclass(frozen=True)
-class Unimported:
-    path: Path
-    artist_slug: str
-    task_type: str  # "media" or "archive", as ImportTask.task_type
-    size: int
-
-
-def _referenced_paths(session: Session) -> set[str]:
-    paths: set[str] = set()
-    for p, thumb in session.execute(select(ImageRecord.path, ImageRecord.thumbnail_path)):
-        paths.add(p)
-        if thumb:
-            paths.add(thumb)
-    paths.update(session.execute(select(PostAttachment.path)).scalars())
-    paths.update(session.execute(
-        select(ImportTask.source_path).where(ImportTask.status.in_(_ACTIVE_TASK_STATUSES))
-    ).scalars())
-    return paths
-
-
-def find_unimported(
-    session: Session, images_root: Path, *, now: float | None = None,
-) -> tuple[list[Unimported], dict]:
-    """Walk the artists' folders and return (files to import, summary).
-
-    The summary counts what was found and what was passed over, and lists the
-    artists with the most to import (bytes first) with a few sample paths each:
-        {"files", "bytes", "archives", "artists": [{slug, files, bytes,
-         samples}], "recent", "not_media", "no_artist"}"""
-    now = time.time() if now is None else now
-    referenced = _referenced_paths(session)
-    slugs = set(session.execute(select(Artist.slug)).scalars())
-    found: list[Unimported] = []
-    passed = {"recent": 0, "not_media": 0, "no_artist": 0}
-    root = Path(images_root)
-    for top in sorted(os.listdir(root)):
-        if top in _STORE_DIRS or top.startswith(("_", ".")):
-            continue
-        top_dir = root / top
-        if not top_dir.is_dir():
-            continue
-        slug = slugify(top)
-        known = slug in slugs
-        for dirpath, dirnames, filenames in os.walk(top_dir):
-            dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
-            for name in sorted(filenames):
-                path = Path(dirpath) / name
-                if (
-                    name.startswith(".") or path.suffix.lower() == ".json"
-                    or is_temp_staging_file(path)
-                ):
-                    continue
-                if str(path) in referenced:
-                    continue
-                try:
-                    st = path.lstat()
-                except OSError:
-                    continue
-                if not stat.S_ISREG(st.st_mode):
-                    continue
-                kind = "archive" if is_archive(path) else ("media" if is_supported(path) else None)
-                if kind is None:
-                    passed["not_media"] += 1
-                elif now - st.st_mtime < MIN_AGE_SECONDS:
-                    passed["recent"] += 1
-                elif not known:
-                    passed["no_artist"] += 1
-                else:
-                    found.append(Unimported(path, slug, kind, st.st_size))
-
-    per_artist: dict[str, dict] = defaultdict(lambda: {"files": 0, "bytes": 0, "samples": []})
-    for u in found:
-        a = per_artist[u.artist_slug]
-        a["files"] += 1
-        a["bytes"] += u.size
-        if len(a["samples"]) < _SAMPLES_PER_ARTIST:
-            a["samples"].append(str(u.path.relative_to(root)))
-    artists = sorted(
-        ({"slug": s, **v} for s, v in per_artist.items()), key=lambda a: -a["bytes"],
-    )[:_ARTISTS_LISTED]
-    summary = {
-        "files": len(found),
-        "bytes": sum(u.size for u in found),
-        "archives": sum(1 for u in found if u.task_type == "archive"),
-        "artists": artists,
-        **passed,
-    }
-    return found, summary
-
-
-def queue_unimported(
-    session: Session, images_root: Path, *, triggered_by: str = "manual",
-) -> dict:
-    """Create a `library` import batch holding every file `find_unimported`
-    lists, and queue it. Returns the same summary plus `batch_id` (None when
-    there was nothing to import)."""
-    from ..tasks.import_file import enqueue_import  # tasks import services
-
-    found, summary = find_unimported(session, images_root)
-    summary["batch_id"] = None
-    if not found:
-        return summary
-    batch = ImportBatch(
-        triggered_by=triggered_by, source_path=str(images_root),
-        scan_mode="library", status="running", total_files=len(found),
-    )
-    session.add(batch)
-    session.flush()
-    tasks = [
-        ImportTask(
-            batch_id=batch.id, source_path=str(u.path), task_type=u.task_type,
-            status="queued", size_bytes=u.size,
-        )
-        for u in found
-    ]
-    session.add_all(tasks)
-    session.commit()
-    for t in tasks:
-        enqueue_import(t.id, t.task_type)
-    summary["batch_id"] = batch.id
-    return summary
-
-
-def folder_owner(
-    session: Session, path: Path, images_root: Path,
-) -> tuple[Artist | None, Source | None]:
-    """The artist whose folder holds `path`, and that artist's source for the
-    platform folder below it when there is exactly one (a Discord artist has a
-    source per channel, so there the post's sidecar decides)."""
-    parts = Path(path).relative_to(images_root).parts
-    artist = session.execute(
-        select(Artist).where(Artist.slug == slugify(parts[0]))
-    ).scalar_one_or_none()
-    if artist is None or len(parts) < 3:
-        return artist, None
-    sources = session.execute(
-        select(Source).where(Source.artist_id == artist.id, Source.platform == parts[1])
-    ).scalars().all()
-    return artist, (sources[0] if len(sources) == 1 else None)

backend/app/tasks/admin.py

@@ -459,29 +459,3 @@ def reclaim_orphaned_attachments_task(self, dry_run: bool = True) -> dict:
         return cleanup_service.reclaim_orphaned_attachments(
             session, images_root=IMAGES_ROOT, dry_run=dry_run,
         )
-
-
-@celery.task(
-    name="backend.app.tasks.admin.import_unimported_files_task",
-    bind=True,
-    autoretry_for=(OperationalError, DBAPIError),
-    retry_backoff=15, retry_backoff_max=180, max_retries=1,
-    # One walk of the artists' folders: a stat per file, so minutes on a large
-    # library on network storage. The limits are the backstop for a wedged
-    # filesystem, not the expected exit.
-    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
-)
-def import_unimported_files_task(self, dry_run: bool = True) -> dict:
-    """Find files in the artists' folders that the library has no record of.
-    dry_run (the default) returns what was found, per artist, without changing
-    anything; apply queues them as a `library` import batch, which attaches each
-    file where it lies and removes the ones the library already holds or the
-    import filters reject."""
-    from ..services import library_scan
-
-    SessionLocal = _sync_session_factory()
-    with SessionLocal() as session:
-        if dry_run:
-            _found, summary = library_scan.find_unimported(session, IMAGES_ROOT)
-            return {**summary, "batch_id": None}
-        return library_scan.queue_unimported(session, IMAGES_ROOT)

backend/app/tasks/backup.py

@@ -8,7 +8,6 @@ BackupRun.
 from __future__ import annotations
 
 import logging
-import time
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -19,7 +18,7 @@ from sqlalchemy.exc import DBAPIError, OperationalError
 from ..celery_app import celery
 from ..config import get_config
 from ..models import BackupRun, ImportSettings
-from ..services import backup_destination, backup_media, backup_s3, backup_service
+from ..services import backup_service
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
 log = logging.getLogger(__name__)
@@ -50,12 +49,7 @@ def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
 )
 def backup_db_task(self, *, tag: str | None = None,
                    triggered_by: str = "manual") -> dict:
-    """Create one DB backup. Returns {'backup_run_id': N}.
-
-    With an off-site destination set up, the dump streams to that bucket and
-    nothing is written here; the row's manifest says where it went
-    (`offsite`: endpoint, bucket, key). Without one, the dump is written to
-    /images/_backups."""
+    """Create one DB backup. Returns {'backup_run_id': N}."""
     SessionLocal = _sync_session_factory()
     cfg = get_config()
     now = datetime.now(UTC)
@@ -70,19 +64,10 @@ def backup_db_task(self, *, tag: str | None = None,
         run_id = row.id
 
     try:
-        with SessionLocal() as session:
-            settings = ImportSettings.load_sync(session)
-            target = backup_destination.saved_target(settings)
-            limit_mbps = settings.backup_upload_limit_mbps
-        if target is not None:
-            result = backup_service.backup_db_offsite(
-                db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps,
-            )
-        else:
-            result = backup_service.backup_db(
-                db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
-                tag=tag, triggered_by=triggered_by,
-            )
+        result = backup_service.backup_db(
+            db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
+            tag=tag, triggered_by=triggered_by,
+        )
     except (SoftTimeLimitExceeded, Exception) as exc:
         with SessionLocal() as session:
             row = session.get(BackupRun, run_id)
@@ -94,23 +79,12 @@ def backup_db_task(self, *, tag: str | None = None,
         row = session.get(BackupRun, run_id)
         row.status = "ok"
         row.finished_at = datetime.now(UTC)
+        row.sql_path = result["sql_path"]
         row.size_bytes = result["size_bytes"]
-        if target is not None:
-            row.manifest = {
-                "ts": result["ts"],
-                "offsite": {
-                    "endpoint": result["endpoint"],
-                    "bucket": result["bucket"],
-                    "key": result["key"],
-                    "parts": result["parts"],
-                },
-            }
-        else:
-            row.sql_path = result["sql_path"]
-            row.manifest = {
-                "manifest_path": result["manifest_path"],
-                "ts": result["ts"],
-            }
+        row.manifest = {
+            "manifest_path": result["manifest_path"],
+            "ts": result["ts"],
+        }
         session.commit()
     return {"backup_run_id": run_id}
 
@@ -268,9 +242,7 @@ def restore_images_task(self, *, source_backup_run_id: int) -> dict:
 def prune_backups() -> dict:
     """Daily Beat. Per-kind retention from ImportSettings.
 
-    Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
-    "files_unlinked": K}. Media runs leave no files here; only their history
-    is trimmed, to the latest MEDIA_RUNS_KEPT.
+    Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
     Tagged rows (tag IS NOT NULL) are never pruned.
     Status='running' / 'restoring' rows are never pruned — the
     recover_stalled_backup_runs sweep flips truly-stuck ones to
@@ -279,13 +251,12 @@ def prune_backups() -> dict:
     Audit 2026-06-02 added the dedicated sweep.)
     """
     SessionLocal = _sync_session_factory()
-    counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0, "files_unlinked": 0}
+    counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
     with SessionLocal() as session:
         s = ImportSettings.load_sync(session)
         for kind, keep in (
             ("db", s.backup_db_keep_last_n),
             ("images", s.backup_images_keep_last_n),
-            ("media", MEDIA_RUNS_KEPT),
         ):
             candidates = session.execute(
                 select(BackupRun)
@@ -329,168 +300,4 @@ def backup_db_nightly() -> dict:
     if now_hour != configured_hour:
         return {"skipped": f"hour={now_hour} != configured={configured_hour}"}
     res = backup_db_task.delay(triggered_by="nightly")
-    out = {"dispatched": res.id}
-    # With an off-site destination, the night's backup is the whole library:
-    # the database dump, then any images and attachments not yet in the bucket.
-    # Both run on the long-maintenance lane, one after the other.
-    with SessionLocal() as session:
-        try:
-            offsite = backup_destination.saved_target(ImportSettings.load_sync(session))
-        except Exception:  # noqa: BLE001 — the media run reports it itself
-            offsite = True
-    if offsite is not None:
-        out["media_dispatched"] = backup_media_task.delay(triggered_by="nightly").id
-    return out
-
-
-# ── Off-site copy of the library's files ─────────────────────────────────────
-
-# Each task invocation sends files for this long, then queues the next chunk
-# of the same run, so other long-maintenance jobs get their turn in between.
-MEDIA_CHUNK_SECONDS = 45 * 60
-MEDIA_RUNS_KEPT = 60
-# A failed file is listed on the run, up to this many; the count covers all.
-MEDIA_FAILURES_LISTED = 20
-
-
-def _offsite_manifest(target) -> dict:
-    return {"endpoint": target.endpoint, "bucket": target.bucket,
-            "key": backup_media.PREFIX}
-
-
-@celery.task(
-    name="backend.app.tasks.backup.backup_media_task",
-    bind=True,
-    autoretry_for=(OperationalError, DBAPIError),
-    retry_backoff=30, retry_backoff_max=300, max_retries=2,
-    # A chunk stops starting files after MEDIA_CHUNK_SECONDS; the limits leave
-    # room for the file in flight, a multi-gigabyte archive at a low upload
-    # limit included.
-    soft_time_limit=21600, time_limit=23400,
-)
-def backup_media_task(self, *, run_id: int | None = None,
-                      triggered_by: str = "manual") -> dict:
-    """Send the library's images and attachments that aren't in the off-site
-    bucket yet, one chunk of a run. Called without `run_id` it starts a run
-    (unless one is already going); each chunk queues the next until nothing is
-    left. A file that can't be sent is listed on the run and tried again by
-    the next run. Returns {'backup_run_id': N, ...}."""
-    SessionLocal = _sync_session_factory()
-    now = datetime.now(UTC)
-    with SessionLocal() as session:
-        if run_id is None:
-            going = session.execute(
-                select(BackupRun.id).where(BackupRun.kind == "media")
-                .where(BackupRun.status == "running").limit(1)
-            ).scalar_one_or_none()
-            if going is not None:
-                return {"backup_run_id": going, "skipped": "a media backup is already running"}
-            row = BackupRun(kind="media", status="running", triggered_by=triggered_by,
-                            started_at=now, last_progress_at=now, size_bytes=0,
-                            manifest={"sent": 0, "failed": 0, "failures": [], "cursor": ""})
-            session.add(row)
-            session.commit()
-            run_id = row.id
-        else:
-            row = session.get(BackupRun, run_id)
-            if row is None or row.status != "running":
-                return {"backup_run_id": run_id, "skipped": "run is no longer running"}
-
-    try:
-        with SessionLocal() as session:
-            settings = ImportSettings.load_sync(session)
-            target = backup_destination.saved_target(settings)
-            limit_mbps = settings.backup_upload_limit_mbps
-        if target is None:
-            raise backup_s3.S3Error("No off-site destination is set up.")
-        more = _send_media_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps))
-    except (SoftTimeLimitExceeded, Exception) as exc:
-        with SessionLocal() as session:
-            row = session.get(BackupRun, run_id)
-            if row is not None:
-                _mark_failed(session, row, exc)
-        raise
-
-    if more:
-        backup_media_task.delay(run_id=run_id)
-        return {"backup_run_id": run_id, "continues": True}
-    with SessionLocal() as session:
-        row = session.get(BackupRun, run_id)
-        m = dict(row.manifest or {})
-        row.status = "ok"
-        row.finished_at = datetime.now(UTC)
-        if m.get("failed"):
-            first = (m.get("failures") or [{}])[0]
-            row.error = (f"{m['failed']} file(s) weren't sent; the next run tries them again. "
-                         f"First: {first.get('path')}: {first.get('error')}")[:2000]
-        session.commit()
-    return {"backup_run_id": run_id, "continues": False}
-
-
-def _send_media_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
-    """Send files until the chunk's time is up or none are left. Progress is
-    committed after every file, so a chunk that dies loses at most the file
-    in flight. Returns True when files remain for another chunk."""
-    tid = backup_media.target_id(target)
-    deadline = time.monotonic() + MEDIA_CHUNK_SECONDS
-    with SessionLocal() as session:
-        row = session.get(BackupRun, run_id)
-        m = dict(row.manifest or {})
-        m["offsite"] = _offsite_manifest(target)
-        cursor = m.get("cursor") or ""
-        while True:
-            batch = backup_media.pending(session, tid, after=cursor)
-            # End the read before any upload starts: a transaction left open
-            # across a long transfer holds back vacuum for its whole length.
-            session.commit()
-            if not batch:
-                m["cursor"] = cursor
-                row.manifest = dict(m)
-                row.last_progress_at = datetime.now(UTC)
-                session.commit()
-                return False
-            for blob in batch:
-                if time.monotonic() > deadline:
-                    m["cursor"] = cursor
-                    row.manifest = dict(m)
-                    session.commit()
-                    return True
-                try:
-                    key = backup_media.send(target, blob, IMAGES_ROOT, pacer)
-                except (backup_s3.S3Error, OSError) as exc:
-                    m["failed"] = m.get("failed", 0) + 1
-                    if len(m.get("failures") or []) < MEDIA_FAILURES_LISTED:
-                        m["failures"] = [*(m.get("failures") or []),
-                                         {"path": blob.path, "error": str(exc)[:300]}]
-                else:
-                    backup_media.record(session, tid, [(blob, key)])
-                    m["sent"] = m.get("sent", 0) + 1
-                    row.size_bytes = (row.size_bytes or 0) + blob.size_bytes
-                cursor = blob.sha256
-                m["cursor"] = cursor
-                row.manifest = dict(m)
-                row.last_progress_at = datetime.now(UTC)
-                session.commit()
-
-
-@celery.task(
-    name="backend.app.tasks.backup.reconcile_media_backup",
-    soft_time_limit=1800, time_limit=2100,
-)
-def reconcile_media_backup() -> dict:
-    """Weekly Beat. Compare what the off-site bucket holds under `media/` with
-    what FabledCurator believes it sent: a missing object is sent again by
-    the next run, and an object already there is not sent twice. Skipped
-    while a media run is going, and when no destination is set up."""
-    SessionLocal = _sync_session_factory()
-    with SessionLocal() as session:
-        target = backup_destination.saved_target(ImportSettings.load_sync(session))
-        if target is None:
-            return {"skipped": "no destination"}
-        going = session.execute(
-            select(BackupRun.id).where(BackupRun.kind == "media")
-            .where(BackupRun.status == "running").limit(1)
-        ).scalar_one_or_none()
-        if going is not None:
-            return {"skipped": "a media backup is running"}
-        return backup_media.reconcile(session, target)
+    return {"dispatched": res.id}

backend/app/tasks/external.py

@@ -24,7 +24,7 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 import redis
-from sqlalchemy import delete, or_, select, update
+from sqlalchemy import delete, select, update
 
 from ..celery_app import celery
 from ..config import get_config
@@ -32,7 +32,6 @@ from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, So
 from ..services.external_fetch import fetch_external
 from ..services.importer import Importer
 from ..services.thumbnailer import Thumbnailer
-from ..utils.staging import is_temp_staging_file
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
 log = logging.getLogger(__name__)
@@ -40,20 +39,14 @@ log = logging.getLogger(__name__)
 IMAGES_ROOT = Path("/images")
 
 # After this many failed attempts a link is dead-lettered (skipped by routine
-# sweeps; the daily recovery retries it, sparingly). Mirrors the ingester ledger.
+# sweeps; an operator recovery still re-attempts). Mirrors the ingester ledger.
 DEAD_LETTER_THRESHOLD = 3
 # Per-fetch read + total budgets now live in external_fetch (a short read
 # timeout fails a stalled host fast; a generous total caps a big-but-flowing
 # download). The celery soft/hard time_limit is the outer backstop above those.
 # Links enqueued per sweep — bounds the burst when a big backfill records many.
 _SWEEP_BATCH = 50
-# Recovery gives a dead link one more attempt once it has been dead for
-# 2**recoveries days (1, 2, 4, 8, 16), and stops after _MAX_RECOVERIES. A host
-# outage gets several later chances; a link that can never succeed (too big to
-# import in time, a file made private) costs a handful of downloads, not three
-# a day.
-_MAX_RECOVERIES = 5
-# A link recovery has given up on is pruned this long after its last attempt.
+# Dead rows older than this are pruned (retention).
 _RETENTION_DAYS = 30
 # Per-host serialize lock. TTL is a safety net for a worker that dies holding it
 # (normal completion/error releases it in `finally`); sized just past the fetch
@@ -61,9 +54,6 @@ _RETENTION_DAYS = 30
 # than one fetch would have taken.
 _LOCK_PREFIX = "fc:extdl_lock:"
 _LOCK_TTL = 2400
-# A link 'downloading' this long has outlived fetch_external_link's hard limit
-# (time_limit=3600), so its worker is gone and the sweep recovers it.
-_STALL_MINUTES = 65
 _SERIALIZE_COUNTDOWN = 120
 _MAX_SERIALIZE_WAITS = 30
 
@@ -116,7 +106,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
                 ExternalLink.id == link_id,
                 ExternalLink.status.in_(("pending", "failed")),
             )
-            .values(status="downloading", started_at=datetime.now(UTC))
+            .values(status="downloading")
             .returning(ExternalLink.id)
         ).first()
         session.commit()
@@ -190,9 +180,6 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
         / str(post.external_post_id) / str(link_id)
     )
     try:
-        # Clear the staging files an interrupted attempt left behind. Complete
-        # files it left stay, and are imported once this fetch succeeds.
-        _discard_partials(post_dir)
         result = fetch_external(host, url, post_dir)
         with SessionLocal() as session:
             link = session.get(ExternalLink, link_id)
@@ -203,7 +190,6 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
                 )
                 _record_failure(link, attempts, result.error or "fetch failed")
                 session.commit()
-                _discard_partials(post_dir)
                 return {"link_id": link_id, "error": result.error}
 
             log.info(
@@ -215,8 +201,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             post = session.get(Post, link.post_id)
             artist = session.get(Artist, post.artist_id)
             source = session.get(Source, post.source_id) if post.source_id else None
-            files = _unimported_files(session, post_dir, first=result.files)
-            image_ids = _route_files(session, files, post, platform, artist, source)
+            image_ids = _route_files(session, result.files, post, platform, artist, source)
             link.status = "downloaded"
             link.last_error = None
             link.completed_at = datetime.now(UTC)
@@ -246,7 +231,6 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             if link is not None and link.status == "downloading":
                 _record_failure(link, attempts, f"{type(exc).__name__}: {exc}")
                 session.commit()
-        _discard_partials(post_dir)
         raise
     finally:
         if lock is not None:
@@ -303,9 +287,10 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
                 post.id, f.name, result.status, len(ids),
             )
         elif result.status == "attached":
-            # Non-art, or an archive with no new images, captured as a
-            # PostAttachment. An archive captured WITHOUT extracting any image
-            # carries the reason, so the "zip but no images" case is visible.
+            # Non-art / archive captured as a PostAttachment (copied into the
+            # store) — drop the on-disk original. An archive captured WITHOUT
+            # extracting any image carries the reason (the recurring "zip but no
+            # images" symptom, #718) — surface it loudly.
             if result.error:
                 log.warning(
                     "extdl archive captured UNEXTRACTED: post=%s file=%s reason=%s",
@@ -313,104 +298,51 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
                 )
             else:
                 log.info("extdl attachment captured: post=%s file=%s", post.id, f.name)
+            f.unlink(missing_ok=True)
         elif result.status == "skipped":
             reason = result.skip_reason.value if result.skip_reason else None
             log.info(
                 "extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason,
             )
+            if reason in ("duplicate_hash", "duplicate_phash"):
+                # Path-safe unlink: external files are imported IN PLACE, so `f`
+                # can BE the canonical record's backing file (same content
+                # re-fetched / a colliding name). Only delete `f` when it is NOT
+                # the existing record's file — otherwise we orphan the record and
+                # playback 404s. (#859)
+                canonical = None
+                if result.image_id is not None:
+                    rec = session.get(ImageRecord, result.image_id)
+                    if rec is not None:
+                        canonical = (IMAGES_ROOT / rec.path).resolve()
+                if canonical != f.resolve():
+                    f.unlink(missing_ok=True)  # a redundant copy — safe to drop
+                else:
+                    log.info(
+                        "extdl keep: %s IS record %s's canonical file — not unlinking",
+                        f.name, result.image_id,
+                    )
         elif result.status == "failed":
             log.warning(
                 "extdl import FAILED: post=%s file=%s error=%s",
                 post.id, f.name, result.error,
             )
+            f.unlink(missing_ok=True)
         else:  # 'refreshed' or any future status — work happened, no new images
             log.info(
                 "extdl import: post=%s file=%s status=%s", post.id, f.name, result.status,
             )
-        # Same policy as the subscription download path: drop the file unless
-        # the library needs it at this path. A file imported in place is kept.
-        importer.discard_after_attach(f, result)
         # The synthesized sidecar has done its job — don't litter the tree.
         Path(str(f) + ".json").unlink(missing_ok=True)
     return image_ids
 
 
-def _unimported_files(session, link_dir: Path, *, first=()) -> list[Path]:
-    """Every complete file under a link's dir, subfolders included, that no
-    ImageRecord points at: what this fetch wrote plus anything an earlier
-    attempt finished writing but never imported. `first` (the fetch's own
-    files) leads the list. Staging files and sidecars are left out."""
-    if not link_dir.is_dir():
-        return []
-    found = {
-        p for p in link_dir.rglob("*")
-        if p.is_file() and not is_temp_staging_file(p) and p.suffix.lower() != ".json"
-    }
-    ordered = [p for p in first if p in found] + sorted(found.difference(first))
-    recorded = set(session.execute(
-        select(ImageRecord.path).where(
-            ImageRecord.path.in_([str(p) for p in ordered])
-        )
-    ).scalars()) if ordered else set()
-    return [p for p in ordered if str(p) not in recorded]
-
-
-def _discard_partials(link_dir: Path) -> int:
-    """Delete the staging files under a link's dir (megatools' `.megatmp.*`, an
-    HTTP fetch's `.part`), then the dirs left empty. Every fetcher writes to a
-    staging name and renames on completion, so a staging file is a transfer
-    that never finished. Complete files stay: they may be the only copy of art
-    whose upstream has since gone. Returns the number of files removed."""
-    if not link_dir.is_dir():
-        return 0
-    removed = 0
-    for p in link_dir.rglob("*"):
-        if not (p.is_file() and is_temp_staging_file(p)):
-            continue
-        try:
-            p.unlink(missing_ok=True)
-            removed += 1
-        except OSError as exc:
-            log.warning("extdl could not remove %s: %s", p, exc)
-    for d in sorted((p for p in link_dir.rglob("*") if p.is_dir()), reverse=True):
-        try:
-            d.rmdir()
-        except OSError:
-            pass  # not empty: it holds a complete file
-    try:
-        link_dir.rmdir()
-    except OSError:
-        pass
-    if removed:
-        log.info("extdl removed %d leftover file(s) from %s", removed, link_dir)
-    return removed
-
-
 @celery.task(name="backend.app.tasks.external.sweep_external_links")
 def sweep_external_links() -> dict:
     """Enqueue a bounded batch of actionable links (pending, or failed below the
-    dead-letter threshold). Driven by the post-download hook and a beat tick.
-
-    First it recovers links whose worker died mid-fetch: a link 'downloading'
-    for longer than the task's hard limit has no worker left, so it counts as a
-    failed attempt and goes back to the retry path (dead-lettered at the
-    threshold like any other failure). The next attempt clears its staging
-    files and imports the complete ones."""
+    dead-letter threshold). Driven by the post-download hook and a beat tick."""
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
-        now = datetime.now(UTC)
-        stale = session.execute(
-            select(ExternalLink).where(
-                ExternalLink.status == "downloading",
-                or_(
-                    ExternalLink.started_at.is_(None),
-                    ExternalLink.started_at < now - timedelta(minutes=_STALL_MINUTES),
-                ),
-            )
-        ).scalars().all()
-        for link in stale:
-            _record_failure(link, link.attempts, "worker stopped mid-fetch")
-        session.commit()
         ids = session.execute(
             select(ExternalLink.id).where(
                 ExternalLink.status.in_(("pending", "failed")),
@@ -419,53 +351,34 @@ def sweep_external_links() -> dict:
         ).scalars().all()
     for link_id in ids:
         fetch_external_link.delay(link_id)
-    return {"enqueued": len(ids), "recovered": len(stale)}
+    return {"enqueued": len(ids)}
 
 
 @celery.task(name="backend.app.tasks.external.recover_external_links")
 def recover_external_links() -> dict:
-    """Recovery sweep: give each dead link one more attempt once it
-    has been dead for 2**recoveries days, then enqueue. A recovered link that
-    fails again goes straight back to dead, and after _MAX_RECOVERIES it stays
-    dead until retention prunes it."""
+    """Recovery sweep (rule #89): reset dead links back to retryable so a stuck
+    host outage or a since-fixed bug gets another pass, then enqueue. Bounded."""
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
-        now = datetime.now(UTC)
-        candidates = session.execute(
-            select(ExternalLink).where(
-                ExternalLink.status == "dead",
-                ExternalLink.recoveries < _MAX_RECOVERIES,
-            )
-        ).scalars().all()
-        recovered = 0
-        for link in candidates:
-            wait = timedelta(days=2 ** link.recoveries)
-            if link.completed_at is not None and link.completed_at > now - wait:
-                continue
-            link.status = "failed"
-            link.attempts = DEAD_LETTER_THRESHOLD - 1
-            link.recoveries += 1
-            recovered += 1
+        session.execute(
+            update(ExternalLink).where(ExternalLink.status == "dead")
+            .values(status="failed", attempts=0, last_error=None)
+        )
         session.commit()
-    return {**sweep_external_links(), "revived": recovered}
+    return sweep_external_links()
 
 
 @celery.task(name="backend.app.tasks.external.prune_external_links")
 def prune_external_links() -> dict:
-    """Retention: delete links recovery has given up on, once their
-    last attempt is _RETENTION_DAYS old, so the ledger doesn't grow unbounded.
-    Downloaded links are kept (they're the record of what we have)."""
+    """Retention (rule #89): delete long-dead links so the ledger doesn't grow
+    unbounded. Downloaded links are kept (they're the record of what we have)."""
     cutoff = datetime.now(UTC) - timedelta(days=_RETENTION_DAYS)
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
         res = session.execute(
             delete(ExternalLink).where(
                 ExternalLink.status == "dead",
-                ExternalLink.recoveries >= _MAX_RECOVERIES,
-                or_(
-                    ExternalLink.completed_at.is_(None),
-                    ExternalLink.completed_at < cutoff,
-                ),
+                ExternalLink.created_at < cutoff,
             )
         )
         session.commit()

backend/app/tasks/import_file.py

@@ -19,8 +19,7 @@ from sqlalchemy.exc import DBAPIError, OperationalError
 
 from ..celery_app import celery
 from ..models import ImportBatch, ImportSettings, ImportTask
-from ..services.importer import Importer, ImportResult
-from ..services.library_scan import folder_owner
+from ..services.importer import Importer
 from ..services.thumbnailer import Thumbnailer
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -46,13 +45,8 @@ def _mark_failed(session, task, error_msg: str) -> None:
     """Best-effort flip of a 'processing' row to 'failed' + batch counter
     increment. Wrapped in its own try because if the DB is what just
     broke, this commit will also fail — that's why the maintenance sweep
-    exists as a backstop.
-
-    The failure being recorded may have left the session's transaction dead
-    (an error mid-flush), so it is rolled back first; otherwise this commit
-    fails on it too and the row stays 'processing'."""
+    exists as a backstop."""
     try:
-        session.rollback()
         task.status = "failed"
         task.error = error_msg
         task.finished_at = datetime.now(UTC)
@@ -171,27 +165,6 @@ def enqueue_import(task_id: int, task_type: str) -> None:
         import_media_file.delay(task_id)
 
 
-def _attach_where_it_lies(session, importer: Importer, path: Path) -> ImportResult:
-    """Import a file that already sits in its artist's folder (a `library`
-    batch, from services.library_scan). It is attached in place with that
-    artist and source. The post record its folder holds (`_post.json`) is filed
-    first, so the post gets its title and date. Afterwards the file is removed
-    if the library didn't keep it: a copy of something it already holds, or a
-    file the import filters reject. A file whose import crashed stays, with the
-    task marked failed: it may be the only copy of an archive the extractor
-    can't read yet."""
-    if not path.is_file():
-        return ImportResult(status="skipped", error="no longer on disk")
-    artist, source = folder_owner(session, path, IMAGES_ROOT)
-    record = path.parent / "_post.json"
-    if artist is not None and record.is_file():
-        importer.upsert_post_record(record, artist=artist, source=source)
-    result = importer.attach_in_place(path, artist=artist, source=source)
-    if result.status != "failed":
-        importer.discard_after_attach(path, result)
-    return result
-
-
 def _do_import(session, task, import_task_id: int) -> dict:
     """Actual work, called from inside the resilience wrapper."""
     settings = ImportSettings.load_sync(session)
@@ -207,10 +180,7 @@ def _do_import(session, task, import_task_id: int) -> dict:
         deep=deep,
     )
 
-    if batch is not None and batch.scan_mode == "library":
-        result = _attach_where_it_lies(session, importer, Path(task.source_path))
-    else:
-        result = importer.import_one(Path(task.source_path))
+    result = importer.import_one(Path(task.source_path))
 
     if result.status in ("imported", "superseded"):
         task.status = "complete"

backend/app/tasks/maintenance.py

@@ -20,12 +20,10 @@ from ..models import (
     ImportSettings,
     ImportTask,
     LibraryAuditRun,
-    PostAttachment,
     Source,
     TaskRun,
 )
 from ..utils.phash import compute_phash
-from ..utils.staging import is_temp_staging_file
 from ._sync_engine import get_sync_engine
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -83,10 +81,13 @@ WIP_BACKFILL_PAGE = 500
 FFPROBE_TIMEOUT_SECONDS = 10
 TASK_RUN_KEEP_OK_SECONDS = 24 * 3600       # 24 h
 TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600  # 7 days
-# Orphaned staging files (utils.staging): cleanup_orphaned_temp_files sweeps the
-# ones a killed write left behind; the min-age guard keeps it from deleting an
-# in-flight download's staging file mid-write.
+# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
+# then os.replace() into place (importer / external_fetch / native_ingest_common /
+# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
+# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
+# keeps it from deleting an in-flight download's staging file mid-write.
 IMAGES_ROOT = Path("/images")
+TEMP_STAGING_SUFFIXES = (".part", ".partial")
 ORPHAN_TEMP_MIN_AGE_HOURS = 6
 
 # Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
@@ -173,9 +174,6 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # legitimately-running backup isn't flagged.
     "backend.app.tasks.backup.backup_images_task": 420,
     "backend.app.tasks.backup.restore_images_task": 420,
-    # One media-backup chunk stops starting files after 45 min but may finish
-    # a large file in flight, up to its 6.5 h hard limit.
-    "backend.app.tasks.backup.backup_media_task": 420,
     # Library audit scans the full library — 2h hard limit.
     "backend.app.tasks.library_audit.scan_library_for_rule": 130,
     # External file-host fetches (mega/gdrive/film packs) can run to the task's
@@ -193,9 +191,6 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # external-fetch entry above — without an override a healthy in-flight walk
     # is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
     "backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
-    # Importing what's on disk stats every file in the artists' folders, and
-    # its hard limit is 40 min for the same wedged-filesystem case. 45 = 40 + 5.
-    "backend.app.tasks.admin.import_unimported_files_task": 45,
     # Head training and the manual head apply run to 65 min (3900s) — past the
     # ml queue's threshold (#4432). 70 = 65 + 5.
     "backend.app.tasks.ml.train_heads": 70,
@@ -203,23 +198,6 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
 }
 
 
-def _import_queue_depth() -> int | None:
-    """Messages waiting in the broker's import queue, or None when the broker
-    can't be read."""
-    try:
-        import redis
-
-        from ..config import get_config
-
-        client = redis.Redis.from_url(
-            get_config().celery_broker_url, socket_timeout=5, socket_connect_timeout=5,
-        )
-        return int(client.llen("import"))
-    except Exception:  # noqa: BLE001 — an unreadable broker means "don't know"
-        log.warning("recover_interrupted_tasks: import queue depth unreadable", exc_info=True)
-        return None
-
-
 @celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
 def recover_interrupted_tasks() -> int:
     """Recover stuck ImportTask rows. Two distinct stuck states:
@@ -235,22 +213,16 @@ def recover_interrupted_tasks() -> int:
        2026-05-24 after a 2224-row zombie pile; archive split out
        2026-05-28.)
 
-    2. 'pending' > 30 min — enqueue-phase crash. scan_directory creates rows
-       with status='pending' (commit), then in a second pass transitions to
-       'queued' and calls .delay() (commit). If the scanner crashes between
-       those two commits, rows are orphaned in 'pending' (never enqueued)
-       with no recovery path — invisible to the 'processing' sweep above.
-       Flip these to 'failed' (not re-enqueue) so the operator drains them
-       via /api/import/retry-failed at their own pace; bulk-re-enqueueing
-       5000+ rows would thundering-herd the import worker.
-
-       A 'queued' row has a broker message, and in a batch larger than half
-       an hour of work it is simply waiting its turn, so its age says
-       nothing. It counts as orphaned only when the broker's import queue
-       is empty: then its message is gone (a broker flush or restart) and
-       nothing will ever run it. When the queue can't be read, queued rows
-       are left alone — a queued row costs nothing to keep, and failing a
-       live one loses its import.
+    2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
+       creates rows with status='pending' (commit), then in a second pass
+       transitions to 'queued' and calls .delay() (commit). If the scanner
+       crashes between those two commits, rows are orphaned in 'pending'
+       (never enqueued) with no recovery path — invisible to the
+       'processing' sweep above. Flagged 2026-05-25 by operator hitting a
+       5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
+       the operator drains them via /api/import/retry-failed at their own
+       pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
+       import worker.
 
     Returns total rows touched (recovered + marked failed).
     """
@@ -327,12 +299,9 @@ def recover_interrupted_tasks() -> int:
         )
         stuck = stuck_result.all()
 
-        orphan_statuses = ["pending"]
-        if _import_queue_depth() == 0:
-            orphan_statuses.append("queued")
         orphan_result = session.execute(
             update(ImportTask)
-            .where(ImportTask.status.in_(orphan_statuses))
+            .where(ImportTask.status.in_(["pending", "queued"]))
             .where(ImportTask.created_at < orphan_cutoff)
             .values(
                 status="failed",
@@ -402,58 +371,22 @@ def cleanup_old_tasks() -> int:
         return result.rowcount or 0
 
 
-def _paths_with_rows(paths: list[str]) -> set[str]:
-    """The subset of `paths` that an image (its file or thumbnail) or an
-    attachment row points at."""
-    owned: set[str] = set()
-    SessionLocal = _sync_session_factory()
-    with SessionLocal() as session:
-        for i in range(0, len(paths), 500):
-            chunk = paths[i:i + 500]
-            owned.update(session.execute(
-                select(ImageRecord.path).where(ImageRecord.path.in_(chunk))
-            ).scalars())
-            owned.update(session.execute(
-                select(ImageRecord.thumbnail_path).where(ImageRecord.thumbnail_path.in_(chunk))
-            ).scalars())
-            owned.update(session.execute(
-                select(PostAttachment.path).where(PostAttachment.path.in_(chunk))
-            ).scalars())
-    return owned
-
-
 @celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
 def cleanup_orphaned_temp_files() -> int:
-    """Delete orphaned staging files (.part, .partial, megatools' .megatmp.*)
-    under the images root, left by a download/import killed mid-write. Only
-    removes files older than ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight
-    download's staging file is never pulled out from under it, and never a
-    file an image or attachment row points at, whatever its name: that file
-    is library content. Returns the count removed."""
+    """Delete orphaned .part/.partial staging files under the images root, left by
+    a download/import killed mid-write. Only removes files older than
+    ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
+    pulled out from under it. Returns the count removed."""
     if not IMAGES_ROOT.is_dir():
         return 0
     cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
-    stale: list[Path] = []
-    for path in IMAGES_ROOT.rglob("*"):
-        if not is_temp_staging_file(path) or not path.is_file():
-            continue
-        try:
-            if path.stat().st_mtime < cutoff:   # fresher may be an active download
-                stale.append(path)
-        except OSError as exc:
-            log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
-    if not stale:
-        return 0
-    owned = _paths_with_rows([str(p) for p in stale])
     removed = 0
-    for path in stale:
-        if str(path) in owned:
-            log.warning(
-                "cleanup_orphaned_temp_files: keeping %s: a library row points at it",
-                path,
-            )
+    for path in IMAGES_ROOT.rglob("*"):
+        if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
             continue
         try:
+            if path.stat().st_mtime >= cutoff:
+                continue          # too fresh — may be an active download
             path.unlink()
             removed += 1
         except OSError as exc:
@@ -845,18 +778,10 @@ def recover_stalled_backup_runs() -> int:
             update(BackupRun)
             .where(BackupRun.status.in_(["running", "restoring"]))
             # db backups/restores are fast (40-min window); images run hours (7h).
-            # A media run spans many chunks and hours on purpose, so its window
-            # runs from its last progress, not from its start.
             .where(
                 or_(
                     and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff),
-                    and_(
-                        BackupRun.kind == "media",
-                        func.coalesce(BackupRun.last_progress_at, BackupRun.started_at)
-                        < slow_cutoff,
-                    ),
-                    and_(BackupRun.kind.not_in(["db", "media"]),
-                         BackupRun.started_at < slow_cutoff),
+                    and_(BackupRun.kind != "db", BackupRun.started_at < slow_cutoff),
                 )
             )
             .values(status="error", finished_at=now, error=msg)

backend/app/utils/staging.py

@@ -1,20 +0,0 @@
-"""Downloader staging files.
-
-Downloads and imports write into a staging name and `os.replace()` it into
-place when the write completes (importer, external_fetch, native_ingest_common,
-attachment_store), so a write killed midway leaves a staging file, never a
-truncated final file. Any other file is complete.
-"""
-from pathlib import Path
-
-TEMP_STAGING_SUFFIXES = (".part", ".partial")
-# megatools stages a Mega download as `.megatmp.<id>` in the target dir.
-TEMP_STAGING_PREFIXES = (".megatmp.",)
-
-
-def is_temp_staging_file(path: Path) -> bool:
-    """True for a downloader's staging file: one that never becomes library
-    content under that name, so a stale one is safe to delete."""
-    return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith(
-        TEMP_STAGING_PREFIXES
-    )

frontend/src/components/settings/BackupCard.vue

@@ -4,18 +4,11 @@
     title="Backups"
     blurb="Database + image backups, schedules and restore."
   >
-      <p v-if="offsite" class="fc-muted text-body-2 mb-4">
-        Database dumps stream to the <strong>{{ offsite.bucket }}</strong> bucket
-        at {{ offsite.endpoint }}, with no copy kept on this server; the bucket's
-        own retention thins old dumps. Images and attachments go to the same
-        bucket's <code>media/</code> folder, below.
-      </p>
-      <p v-else class="fc-muted text-body-2 mb-4">
+      <p class="fc-muted text-body-2 mb-4">
         pg_dump for the database (fast, nightly-schedulable);
         tar+zstd for the images (slow, manual only). Files live in
         <code>/images/_backups/</code>. Tag a backup to protect it
-        from autoprune. Set up an off-site destination above to send
-        database dumps off this server instead.
+        from autoprune.
       </p>
 
       <v-alert
@@ -47,10 +40,6 @@
           @update:model-value="onSettingChange('backup_db_keep_last_n', Number($event))"
         />
       </div>
-      <p v-if="offsite" class="fc-muted text-caption mb-3">
-        Nightly sends the database, then any new images and attachments. Keep
-        last applies to this list and to any dumps still on this server.
-      </p>
       <v-btn
         color="accent" variant="flat" rounded="pill"
         prepend-icon="mdi-database-arrow-up" class="mb-3"
@@ -67,38 +56,6 @@
 
       <v-divider class="my-4" />
 
-      <!-- Off-site: images and attachments as one file per fingerprint -->
-      <template v-if="offsite">
-        <h3 class="fc-section-title">Images and attachments</h3>
-        <p class="fc-muted text-body-2 mb-2">
-          Each file goes to the bucket's <code>media/</code> folder once, named by
-          its fingerprint, and the bucket checks it against the fingerprint taken
-          at import. Only files not sent yet go in each run. With Nightly on, they
-          follow the database dump every night.
-        </p>
-        <div v-if="coverage" class="mb-3">
-          <div class="text-body-2 mb-1" style="font-variant-numeric: tabular-nums;">
-            {{ coverage.files_sent.toLocaleString() }} of
-            {{ coverage.files_total.toLocaleString() }} files in the bucket
-            ({{ humanBytes(coverage.bytes_sent) }} of {{ humanBytes(coverage.bytes_total) }})
-          </div>
-          <v-progress-linear
-            :model-value="coveragePct" color="accent" height="6" rounded
-          />
-        </div>
-        <v-btn
-          color="accent" variant="flat" rounded="pill"
-          prepend-icon="mdi-cloud-upload-outline" class="mb-3"
-          :loading="mediaTriggering" :disabled="mediaRunning"
-          @click="onTrigger('media')"
-        >Send new files now</v-btn>
-        <span v-if="mediaRunning" class="text-caption fc-muted ml-2">
-          A run is going: {{ runningSent.toLocaleString() }} files sent so far.
-        </span>
-        <BackupRunsTable :runs="store.mediaRuns" />
-      </template>
-
-      <template v-else>
       <!-- Images section -->
       <h3 class="fc-section-title">Images</h3>
       <div class="fc-settings-row mb-3">
@@ -125,7 +82,6 @@
         @delete="onDelete"
         @tag="onTag"
       />
-      </template>
     <BackupConfirmModal
       v-model="confirmOpen"
       :action="confirmAction"
@@ -143,7 +99,6 @@ import { computed, onMounted, onUnmounted, ref } from 'vue'
 import { useBackupStore } from '../../stores/backup.js'
 import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
 import MaintenanceTile from '../common/MaintenanceTile.vue'
-import { humanBytes } from '../../utils/bytes.js'
 import BackupRunsTable from './BackupRunsTable.vue'
 
 const store = useBackupStore()
@@ -155,21 +110,6 @@ const settings = computed(() => store.settings || {
   backup_images_keep_last_n: 3,
 })
 
-// The off-site destination, once it is fully set up: database dumps go there.
-const offsite = computed(() => (
-  store.destination?.configured ? store.destination : null
-))
-
-const coverage = computed(() => store.mediaCoverage)
-const coveragePct = computed(() => {
-  const c = coverage.value
-  return c && c.files_total ? (100 * c.files_sent) / c.files_total : 0
-})
-const runningMedia = computed(() => store.mediaRuns.find((r) => r.status === 'running'))
-const mediaRunning = computed(() => Boolean(runningMedia.value))
-const runningSent = computed(() => runningMedia.value?.manifest?.sent || 0)
-
-const mediaTriggering = ref(false)
 const dbTriggering = ref(false)
 const imagesTriggering = ref(false)
 const confirmOpen = ref(false)
@@ -179,24 +119,14 @@ const confirmRunId = ref(0)
 const confirmDescription = ref('')
 
 let pollId = null
-let polls = 0
-// Coverage counts every library file, so it refreshes every sixth poll
-// (30 s) rather than with the run lists.
-const COVERAGE_EVERY = 6
 function pollOnce() {
   if (document.hidden) return
   store.loadRuns('db')
-  if (offsite.value) {
-    store.loadRuns('media')
-    if (polls++ % COVERAGE_EVERY === 0) store.loadMediaCoverage()
-  } else {
-    store.loadRuns('images')
-  }
+  store.loadRuns('images')
 }
 
 onMounted(() => {
   store.loadSettings()
-  if (!store.destination) store.loadDestination().catch(() => {})
   pollOnce()
   pollId = setInterval(pollOnce, 5000)
 })
@@ -205,7 +135,7 @@ onUnmounted(() => {
 })
 
 async function onTrigger(kind) {
-  const triggeringRef = { db: dbTriggering, images: imagesTriggering, media: mediaTriggering }[kind]
+  const triggeringRef = kind === 'db' ? dbTriggering : imagesTriggering
   triggeringRef.value = true
   try {
     await store.triggerBackup(kind)

frontend/src/components/settings/BackupDestinationCard.vue

@@ -1,217 +0,0 @@
-<!--
-  Off-site backup destination: an S3 bucket (Fabled Archives or any S3
-  service) that backups are sent to. The five fields only work together, so
-  they save as one with an explicit button rather than field by field. The
-  secret key is write-only: once saved it is never shown again, and leaving the
-  field blank keeps it. Test destination lists the bucket with what is typed
-  in the form (saved or not) and writes nothing.
--->
-<template>
-  <MaintenanceTile
-    icon="mdi-cloud-upload-outline"
-    title="Off-site backup destination"
-    blurb="The S3 bucket (Fabled Archives or any S3 service) that backups are sent to."
-  >
-    <p class="fc-muted text-body-2 mb-3">
-      Backups sent here survive losing this server. FabledCurator only adds to the
-      bucket and never deletes from it, so a key that may add and read objects but
-      not delete them is enough, and is the safer key to use. Account logins are
-      never sent.
-    </p>
-
-    <v-alert
-      v-if="loadError" type="warning" variant="tonal" density="compact" class="mb-3"
-    >{{ loadError }}</v-alert>
-
-    <v-text-field
-      v-model="form.endpoint" label="Address"
-      placeholder="http://archives.example:8081" density="compact"
-      hint="Include the port the S3 service listens on." persistent-hint
-      class="mb-3" :disabled="busy"
-    />
-    <div class="fc-dest-row mb-3">
-      <v-text-field
-        v-model="form.bucket" label="Bucket" density="compact" hide-details
-        :disabled="busy"
-      />
-      <v-text-field
-        v-model="form.region" label="Region" density="compact" hide-details
-        class="fc-dest-region" :disabled="busy"
-      />
-    </div>
-    <v-text-field
-      v-model="form.access_key" label="Access key" density="compact" hide-details
-      autocomplete="off" class="mb-3" :disabled="busy"
-    />
-    <v-text-field
-      v-model="form.secret_key" label="Secret key" type="password"
-      autocomplete="new-password" density="compact"
-      :placeholder="secretSet ? 'Saved. Leave blank to keep it.' : ''"
-      :persistent-placeholder="secretSet"
-      :hint="secretSet ? 'A saved secret is never shown again.' : ''" persistent-hint
-      class="mb-3" :disabled="busy"
-    />
-    <v-text-field
-      v-model.number="form.upload_limit_mbps" label="Upload limit (Mbit/s)"
-      type="number" min="0" density="compact" class="fc-dest-limit mb-3"
-      hint="0 for no limit. A lower limit leaves more of the link free, and makes each upload take longer."
-      persistent-hint :disabled="busy"
-    />
-
-    <div class="d-flex align-center flex-wrap mb-2" style="gap: 8px;">
-      <v-btn
-        size="small" variant="flat" color="accent" rounded="pill"
-        prepend-icon="mdi-content-save" :loading="saving"
-        :disabled="busy || !dirty" @click="onSave"
-      >Save</v-btn>
-      <v-btn
-        size="small" variant="tonal" rounded="pill"
-        prepend-icon="mdi-lan-connect" :loading="testing"
-        :disabled="busy || !form.endpoint" @click="onTest"
-      >Test destination</v-btn>
-      <v-btn
-        v-if="configured" size="small" variant="text" rounded="pill" color="error"
-        :disabled="busy" @click="confirmRemove = true"
-      >Remove</v-btn>
-      <span v-if="savedNote" class="text-caption text-success">{{ savedNote }}</span>
-    </div>
-
-    <v-alert
-      v-if="testResult" :type="testResult.ok ? 'success' : 'error'"
-      variant="tonal" density="compact" class="mb-1"
-    >{{ testResult.message }}</v-alert>
-    <p v-if="!configured && !testResult" class="fc-muted text-caption mb-0">
-      No destination is set up, so nothing is sent off this server yet.
-    </p>
-    <p v-else-if="configured && !testResult" class="fc-muted text-caption mb-0">
-      Database backups and the library's images and attachments go to this bucket.
-      Set when they run under Backups below.
-    </p>
-
-    <v-dialog v-model="confirmRemove" max-width="440">
-      <v-card>
-        <v-card-title class="text-h6">Remove the destination?</v-card-title>
-        <v-card-text class="text-body-2">
-          FabledCurator forgets the address and both keys and stops sending
-          backups there. What was already sent stays in the bucket.
-        </v-card-text>
-        <v-card-actions>
-          <v-spacer />
-          <v-btn variant="text" @click="confirmRemove = false">Cancel</v-btn>
-          <v-btn color="error" variant="flat" :loading="saving" @click="onRemove">
-            Remove
-          </v-btn>
-        </v-card-actions>
-      </v-card>
-    </v-dialog>
-  </MaintenanceTile>
-</template>
-
-<script setup>
-import { computed, onMounted, reactive, ref } from 'vue'
-
-import { useBackupStore } from '../../stores/backup.js'
-import MaintenanceTile from '../common/MaintenanceTile.vue'
-
-const store = useBackupStore()
-
-const FIELDS = ['endpoint', 'bucket', 'access_key', 'region']
-const DEFAULT_LIMIT = 200
-const form = reactive({
-  endpoint: '', bucket: '', access_key: '', region: 'us-east-1', secret_key: '',
-  upload_limit_mbps: DEFAULT_LIMIT,
-})
-const saved = ref(null)
-const loadError = ref('')
-const saving = ref(false)
-const testing = ref(false)
-const testResult = ref(null)
-const savedNote = ref('')
-const confirmRemove = ref(false)
-
-const busy = computed(() => saving.value || testing.value)
-const secretSet = computed(() => Boolean(saved.value?.secret_set))
-const configured = computed(() => Boolean(saved.value?.configured))
-const dirty = computed(() => (
-  !saved.value
-  || form.secret_key.trim() !== ''
-  || FIELDS.some((f) => (form[f] || '').trim() !== (saved.value[f] || ''))
-  || form.upload_limit_mbps !== saved.value.upload_limit_mbps
-))
-
-function apply(dest) {
-  saved.value = dest
-  for (const f of FIELDS) form[f] = dest[f] || (f === 'region' ? 'us-east-1' : '')
-  form.upload_limit_mbps = dest.upload_limit_mbps ?? DEFAULT_LIMIT
-  form.secret_key = ''
-}
-
-function fields() {
-  const out = {}
-  for (const f of FIELDS) out[f] = form[f]
-  if (form.secret_key.trim()) out.secret_key = form.secret_key
-  const limit = Number(form.upload_limit_mbps)
-  if (Number.isInteger(limit) && limit >= 0) out.upload_limit_mbps = limit
-  return out
-}
-
-function messageOf(e) {
-  return e?.body?.detail || e?.message || 'The request failed.'
-}
-
-onMounted(async () => {
-  try {
-    apply(await store.loadDestination())
-  } catch (e) {
-    loadError.value = `Couldn't load the destination: ${messageOf(e)}`
-  }
-})
-
-async function onSave() {
-  saving.value = true
-  savedNote.value = ''
-  testResult.value = null
-  try {
-    apply(await store.saveDestination(fields()))
-    savedNote.value = 'Saved.'
-  } catch (e) {
-    testResult.value = { ok: false, message: `Not saved: ${messageOf(e)}` }
-  } finally {
-    saving.value = false
-  }
-}
-
-async function onTest() {
-  testing.value = true
-  savedNote.value = ''
-  testResult.value = null
-  try {
-    testResult.value = await store.testDestination(fields())
-  } catch (e) {
-    testResult.value = { ok: false, message: messageOf(e) }
-  } finally {
-    testing.value = false
-  }
-}
-
-async function onRemove() {
-  saving.value = true
-  try {
-    apply(await store.removeDestination())
-    testResult.value = null
-    savedNote.value = 'Removed.'
-    confirmRemove.value = false
-  } catch (e) {
-    testResult.value = { ok: false, message: `Not removed: ${messageOf(e)}` }
-  } finally {
-    saving.value = false
-  }
-}
-</script>
-
-<style scoped>
-.fc-dest-row { display: flex; gap: 12px; flex-wrap: wrap; }
-.fc-dest-row > :first-child { flex: 1 1 200px; }
-.fc-dest-region { flex: 0 1 160px; }
-.fc-dest-limit { max-width: 260px; }
-</style>

frontend/src/components/settings/BackupRunsTable.vue

@@ -4,7 +4,6 @@
       <tr>
         <th>When</th>
         <th>Status</th>
-        <th>Saved to</th>
         <th class="text-right">Duration</th>
         <th class="text-right">Size</th>
         <th>Tag</th>
@@ -21,15 +20,6 @@
             {{ statusIcon(r.status) }}
           </v-icon>
           {{ r.status }}
-          <div
-            v-if="r.error" class="fc-run-error text-caption text-error" :title="r.error"
-          >{{ r.error }}</div>
-        </td>
-        <td :title="r.offsite ? `${r.offsite.endpoint} · ${r.offsite.key}` : ''">
-          <template v-if="r.offsite">
-            <v-icon size="small" class="mr-1">mdi-cloud-check-outline</v-icon>{{ r.offsite.bucket }}
-          </template>
-          <span v-else class="fc-muted">This server</span>
         </td>
         <td class="text-right fc-tabular">{{ formatDuration(r) }}</td>
         <td class="text-right fc-tabular">{{ formatBytes(r.size_bytes) }}</td>
@@ -40,8 +30,7 @@
           <span v-else class="fc-muted">—</span>
         </td>
         <td class="text-right">
-          <span v-if="r.offsite" class="fc-muted">—</span>
-          <KebabMenu v-else :label="`Actions for backup ${r.id}`">
+          <KebabMenu :label="`Actions for backup ${r.id}`">
             <v-list-item
               :title="r.tag ? 'Untag' : 'Tag…'"
               @click="$emit('tag', r)"
@@ -58,7 +47,7 @@
         </td>
       </tr>
       <tr v-if="!runs.length">
-        <td colspan="7" class="text-center fc-muted py-4">No backups yet.</td>
+        <td colspan="6" class="text-center fc-muted py-4">No backups yet.</td>
       </tr>
     </tbody>
   </v-table>
@@ -108,7 +97,4 @@ function formatRelative(iso) {
 <style scoped>
 .fc-backup-table { background: transparent; }
 .fc-tabular { font-variant-numeric: tabular-nums; }
-.fc-run-error {
-  max-width: 320px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
-}
 </style>

frontend/src/components/settings/LibraryImportCard.vue

@@ -1,150 +0,0 @@
-<template>
-  <!-- Files a download wrote into an artist's folder but never imported: a run
-       that stopped between writing and importing leaves them there, and the
-       walk that marked them seen never returns. Preview lists them per artist;
-       Import queues them as an import batch that attaches each file where it
-       lies. -->
-  <MaintenanceTile
-    icon="mdi-folder-download"
-    title="Import files the library is missing"
-    blurb="Find art in the artists' folders that was downloaded but never imported, and import it where it is."
-    :open="applying || previewing"
-  >
-    <p class="text-body-2 mb-3">
-      A download writes its files into the artist's folder, then imports them.
-      A run that stops in between leaves complete files that no image points
-      at, and later downloads skip them as already seen. This walks every
-      artist's folder for images, videos and archives the library has
-      <strong>no record of</strong>. <strong>Preview</strong> first;
-      <strong>Import</strong> runs them through the normal import where they
-      lie. A file that turns out to be a copy of something already in the
-      library, or that the import filters reject, is removed from disk. Files
-      changed in the last day are left alone, so a download still running is
-      never caught mid-write.
-    </p>
-
-    <div class="d-flex align-center flex-wrap" style="gap: 12px;">
-      <v-btn
-        color="primary" variant="tonal" rounded="pill"
-        :loading="previewing" :disabled="applying" @click="preview"
-      >
-        <v-icon start>mdi-magnify</v-icon> Preview
-      </v-btn>
-      <v-btn
-        color="primary" rounded="pill"
-        :loading="applying"
-        :disabled="previewing || !canApply"
-        @click="confirmOpen = true"
-      >
-        <v-icon start>mdi-folder-download</v-icon> Import
-      </v-btn>
-    </div>
-
-    <v-alert
-      v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
-      density="comfortable"
-    >
-      <span v-if="applied && summary.batch_id">
-        Queued {{ summary.files }} file(s) ({{ humanBytes(summary.bytes) }}) for
-        import. Progress shows in the import queue below and on Activity.
-      </span>
-      <span v-else-if="hasWork">
-        {{ summary.files }} file(s), {{ humanBytes(summary.bytes) }}, are on
-        disk but not in the library<span v-if="summary.archives">, including
-        {{ summary.archives }} archive(s)</span>.
-        Click <strong>Import</strong> to bring them in.
-      </span>
-      <span v-else>Nothing to import: every file in the artists' folders is in the library.</span>
-
-      <div v-if="passedOver" class="mt-1 text-caption">
-        Left alone: {{ summary.recent }} changed in the last day,
-        {{ summary.not_media }} that aren't images, videos or archives, and
-        {{ summary.no_artist }} in folders that match no artist.
-      </div>
-    </v-alert>
-
-    <v-table v-if="hasWork && !applied" density="compact" class="mt-3">
-      <thead>
-        <tr>
-          <th>Artist</th>
-          <th class="text-right">Files</th>
-          <th class="text-right">Size</th>
-          <th>For example</th>
-        </tr>
-      </thead>
-      <tbody>
-        <tr v-for="a in summary.artists" :key="a.slug">
-          <td>{{ a.slug }}</td>
-          <td class="text-right">{{ a.files }}</td>
-          <td class="text-right">{{ humanBytes(a.bytes) }}</td>
-          <td class="text-caption text-medium-emphasis fc-sample">{{ a.samples[0] }}</td>
-        </tr>
-      </tbody>
-    </v-table>
-
-    <QueueStatusBar queue="import" queue-label="Import" />
-
-    <v-dialog v-model="confirmOpen" max-width="440">
-      <v-card>
-        <v-card-title>Import {{ summary?.files ?? 0 }} file(s)?</v-card-title>
-        <v-card-text class="text-body-2">
-          This imports <strong>{{ summary?.files ?? 0 }}</strong> file(s)
-          ({{ humanBytes(summary?.bytes) }}) where they lie. Any that turn out
-          to be copies of art already in the library, or that the import
-          filters reject, are removed from disk.
-        </v-card-text>
-        <v-card-actions>
-          <v-spacer />
-          <v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
-          <v-btn color="primary" @click="apply">Import</v-btn>
-        </v-card-actions>
-      </v-card>
-    </v-dialog>
-  </MaintenanceTile>
-</template>
-
-<script setup>
-import { computed, ref } from 'vue'
-
-import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
-import { humanBytes } from '../../utils/bytes.js'
-import MaintenanceTile from '../common/MaintenanceTile.vue'
-import QueueStatusBar from './QueueStatusBar.vue'
-
-const confirmOpen = ref(false)
-
-// The walk stats every file in the artists' folders, so on a large library on
-// network storage it can take many minutes. 900 polls × 2s = 30 min, the
-// task's own soft limit.
-const { previewing, applying, summary, applied, preview, apply: applyTask } = useMaintenanceTask({
-  endpoint: '/api/admin/maintenance/import-unimported',
-  storageKey: 'fc.maint.importUnimported',
-  appliedToast: 'Files queued for import',
-  maxPolls: 900,
-})
-
-const hasWork = computed(() => !!summary.value && summary.value.files > 0)
-const canApply = computed(() => hasWork.value && !applied.value)
-const passedOver = computed(() => {
-  const s = summary.value
-  return !!s && (s.recent > 0 || s.not_media > 0 || s.no_artist > 0)
-})
-const summaryType = computed(() => {
-  if (applied.value) return 'success'
-  return hasWork.value ? 'info' : 'success'
-})
-
-function apply () {
-  confirmOpen.value = false
-  applyTask()
-}
-</script>
-
-<style scoped>
-.fc-sample {
-  max-width: 320px;
-  overflow: hidden;
-  text-overflow: ellipsis;
-  white-space: nowrap;
-}
-</style>

frontend/src/components/settings/MaintenancePanel.vue

@@ -63,11 +63,8 @@
 
     <section class="fc-section">
       <h3 class="fc-section__title">Storage</h3>
-      <p class="fc-section__hint">
-        Database + image backups and restore, and where backups are sent off this server.
-      </p>
+      <p class="fc-section__hint">Database + image backups and restore.</p>
       <div class="fc-tile-stack">
-        <BackupDestinationCard />
         <BackupCard />
       </div>
     </section>
@@ -96,7 +93,6 @@ import PostAssociationsCard from './PostAssociationsCard.vue'
 import GpuAgentCard from './GpuAgentCard.vue'
 import AliasTable from './AliasTable.vue'
 import BackupCard from './BackupCard.vue'
-import BackupDestinationCard from './BackupDestinationCard.vue'
 import { useSystemActivityStore } from '../../stores/systemActivity.js'
 
 // Poll queue depths so each task tile's QueueStatusBar shows live pending

frontend/src/stores/backup.js

@@ -8,15 +8,13 @@ export const useBackupStore = defineStore('backup', () => {
 
   const dbRuns = ref([])
   const imagesRuns = ref([])
-  const mediaRuns = ref([])
   const settings = ref(null)
-  const loading = ref({ dbRuns: false, imagesRuns: false, mediaRuns: false, settings: false })
+  const loading = ref({ dbRuns: false, imagesRuns: false, settings: false })
   const lastError = ref(null)
-  const RUNS = { db: dbRuns, images: imagesRuns, media: mediaRuns }
 
   async function loadRuns(kind) {
-    const targetRef = RUNS[kind]
-    const loadingKey = `${kind}Runs`
+    const targetRef = kind === 'db' ? dbRuns : imagesRuns
+    const loadingKey = kind === 'db' ? 'dbRuns' : 'imagesRuns'
     loading.value[loadingKey] = true
     lastError.value = null
     try {
@@ -103,47 +101,9 @@ export const useBackupStore = defineStore('backup', () => {
     }
   }
 
-  // Off-site destination (an S3 bucket). The secret key goes up but never
-  // comes back: the server answers `secret_set` instead.
-  const destination = ref(null)
-
-  async function loadDestination() {
-    destination.value = await api.get('/api/system/backup/destination')
-    return destination.value
-  }
-
-  async function saveDestination(fields) {
-    destination.value = await api.patch('/api/system/backup/destination', { body: fields })
-    return destination.value
-  }
-
-  async function removeDestination() {
-    destination.value = await api.delete('/api/system/backup/destination')
-    return destination.value
-  }
-
-  // Resolves to {ok, message}; an unreachable bucket is a result, not a throw.
-  async function testDestination(fields) {
-    return api.post('/api/system/backup/destination/test', { body: fields })
-  }
-
-  // How much of the library is in the off-site bucket:
-  // {configured, files_total, files_sent, bytes_total, bytes_sent}.
-  const mediaCoverage = ref(null)
-
-  async function loadMediaCoverage() {
-    try {
-      mediaCoverage.value = await api.get('/api/system/backup/media')
-    } catch (e) {
-      lastError.value = e.message
-    }
-  }
-
   return {
-    dbRuns, imagesRuns, mediaRuns, mediaCoverage, loadMediaCoverage,
-    settings, loading, lastError,
+    dbRuns, imagesRuns, settings, loading, lastError,
     loadRuns, triggerBackup, restore, deleteRun, setTag,
     loadSettings, patchSettings,
-    destination, loadDestination, saveDestination, removeDestination, testDestination,
   }
 })

frontend/src/views/CleanupView.vue

@@ -22,15 +22,13 @@
       <h3 class="fc-section__title">Duplicates &amp; leftovers</h3>
       <p class="fc-section__hint">
         Tidy post records, duplicates, locked-preview leftovers and attachments
-        that outlived what they belonged to, and import downloads that never
-        made it into the library.
+        that outlived what they belonged to.
       </p>
       <div class="fc-tile-grid">
         <PostMaintenanceCard />
         <VideoDedupCard />
         <GatedPurgeCard />
         <AttachmentReclaimCard />
-        <LibraryImportCard />
       </div>
     </section>
 
@@ -65,7 +63,6 @@ import PostMaintenanceCard from '../components/settings/PostMaintenanceCard.vue'
 import VideoDedupCard from '../components/settings/VideoDedupCard.vue'
 import GatedPurgeCard from '../components/settings/GatedPurgeCard.vue'
 import AttachmentReclaimCard from '../components/settings/AttachmentReclaimCard.vue'
-import LibraryImportCard from '../components/settings/LibraryImportCard.vue'
 import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
 import DangerZoneCard from '../components/settings/DangerZoneCard.vue'
 </script>

tests/test_api_admin.py

@@ -708,22 +708,3 @@ async def test_trigger_reclaim_attachments_threads_apply(client, monkeypatch):
     )
     assert resp.status_code == 202
     assert calls[0][1] == {"dry_run": False}
-
-
-@pytest.mark.asyncio
-async def test_trigger_import_unimported_defaults_to_preview(client, monkeypatch):
-    """An empty body previews: the apply removes files the library rejects."""
-    from backend.app.tasks import admin as admin_tasks
-
-    calls = []
-    monkeypatch.setattr(
-        admin_tasks.import_unimported_files_task, "delay", _fake_delay(calls)
-    )
-    resp = await client.post("/api/admin/maintenance/import-unimported", json={})
-    assert resp.status_code == 202
-    assert calls[0][1] == {"dry_run": True}
-
-    resp = await client.post(
-        "/api/admin/maintenance/import-unimported", json={"dry_run": False},
-    )
-    assert calls[1][1] == {"dry_run": False}

tests/test_api_system_backup.py

@@ -232,27 +232,6 @@ async def test_restore_rejects_non_ok_status(client, db):
     assert body["error"] == "not_restorable"
 
 
-@pytest.mark.asyncio
-async def test_an_off_site_dump_is_not_restored_from_this_screen(client, db):
-    db.add(BackupRun(
-        kind="db", status="ok", tag=None, triggered_by="nightly",
-        started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
-        manifest={"offsite": {"endpoint": "http://archives.example:8081",
-                              "bucket": "curator", "key": "database/x.dump"}},
-    ))
-    await db.commit()
-    run = (await (await client.get("/api/system/backup/runs?kind=db&limit=1")).get_json())[
-        "runs"][0]
-    assert run["offsite"]["bucket"] == "curator"
-
-    resp = await client.post(
-        f"/api/system/backup/runs/{run['id']}/restore",
-        json={"confirm": f"restore-db-{run['id']}"},
-    )
-    assert resp.status_code == 400
-    assert (await resp.get_json())["error"] == "not_restorable"
-
-
 # --- DELETE /runs/<id> ----------------------------------------------
 
 
@@ -343,156 +322,3 @@ async def test_patch_settings_rejects_keep_n_below_min(client):
         json={"backup_db_keep_last_n": 0},
     )
     assert resp.status_code == 400
-
-
-# --- /destination ---------------------------------------------------
-
-
-@pytest.fixture
-def _sealing_key(tmp_path, monkeypatch):
-    from backend.app.services import backup_destination
-    monkeypatch.setattr(backup_destination, "KEY_PATH", tmp_path / "credential_key.b64")
-
-
-_DEST = {
-    "endpoint": "http://archives.example:8081/",
-    "bucket": "curator",
-    "access_key": "AKEXAMPLE",
-    "region": "",
-    "secret_key": "s3cr3t-example",
-}
-
-
-@pytest.mark.asyncio
-async def test_destination_saves_and_never_returns_the_secret(client, db_sync, _sealing_key):
-    from backend.app.models import ImportSettings
-
-    resp = await client.patch("/api/system/backup/destination", json=_DEST)
-    assert resp.status_code == 200
-    body = await resp.get_json()
-    assert body == {
-        "endpoint": "http://archives.example:8081", "bucket": "curator",
-        "access_key": "AKEXAMPLE", "region": "us-east-1",
-        "secret_set": True, "configured": True, "upload_limit_mbps": 200,
-    }
-    assert "s3cr3t-example" not in str(await (await client.get(
-        "/api/system/backup/destination")).get_json())
-    row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
-    db_sync.refresh(row)
-    assert row.backup_s3_secret_sealed and "s3cr3t-example" not in row.backup_s3_secret_sealed
-
-    from backend.app.services import backup_destination
-    assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
-
-
-@pytest.mark.asyncio
-async def test_destination_blank_secret_keeps_the_saved_one(client, db_sync, _sealing_key):
-    from backend.app.models import ImportSettings
-    from backend.app.services import backup_destination
-
-    await client.patch("/api/system/backup/destination", json=_DEST)
-    resp = await client.patch(
-        "/api/system/backup/destination", json={**_DEST, "bucket": "other", "secret_key": ""},
-    )
-    assert (await resp.get_json())["bucket"] == "other"
-    row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
-    db_sync.refresh(row)
-    assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("patch, word", [
-    ({"endpoint": "archives.example:8081"}, "http://"),
-    ({"endpoint": "ftp://archives.example"}, "http://"),
-    ({"bucket": "curator/sub"}, "bucket"),
-    ({"access_key": 5}, "access_key"),
-])
-async def test_destination_rejects_what_cant_be_an_s3_target(client, _sealing_key, patch, word):
-    resp = await client.patch("/api/system/backup/destination", json={**_DEST, **patch})
-    assert resp.status_code == 400
-    assert word in (await resp.get_json())["detail"]
-
-
-@pytest.mark.asyncio
-async def test_destination_remove_forgets_everything(client, _sealing_key):
-    await client.patch("/api/system/backup/destination", json=_DEST)
-    resp = await client.delete("/api/system/backup/destination")
-    assert await resp.get_json() == {
-        "endpoint": "", "bucket": "", "access_key": "", "region": "us-east-1",
-        "secret_set": False, "configured": False, "upload_limit_mbps": 200,
-    }
-
-
-@pytest.mark.asyncio
-async def test_destination_test_uses_the_form_and_the_saved_secret(
-    client, _sealing_key, monkeypatch,
-):
-    from backend.app.services import backup_destination
-
-    seen = []
-
-    def fake_check(target):
-        seen.append(target)
-        return {"ok": True, "message": "Reached it."}
-
-    monkeypatch.setattr(backup_destination, "check_target", fake_check)
-    await client.patch("/api/system/backup/destination", json=_DEST)
-    resp = await client.post(
-        "/api/system/backup/destination/test",
-        json={**_DEST, "bucket": "typed-not-saved", "secret_key": ""},
-    )
-    assert await resp.get_json() == {"ok": True, "message": "Reached it."}
-    assert seen[0].bucket == "typed-not-saved"
-    assert seen[0].secret_key == "s3cr3t-example"
-    assert seen[0].endpoint == "http://archives.example:8081"
-
-
-@pytest.mark.asyncio
-async def test_destination_test_names_what_is_missing(client, _sealing_key):
-    await client.delete("/api/system/backup/destination")
-    resp = await client.post(
-        "/api/system/backup/destination/test",
-        json={"endpoint": "http://archives.example:8081", "bucket": "curator"},
-    )
-    body = await resp.get_json()
-    assert body["ok"] is False
-    assert body["message"] == "Not tested: fill in the access key, secret key."
-
-
-@pytest.mark.asyncio
-async def test_destination_test_reports_an_unreachable_bucket_as_a_result(
-    client, _sealing_key, monkeypatch,
-):
-    from backend.app.services import backup_destination
-    from backend.app.services.backup_s3 import S3Error
-
-    def refuse(target, **kw):
-        raise S3Error("Couldn't reach http://archives.example:8081: nothing is listening there.")
-
-    monkeypatch.setattr(backup_destination, "list_objects", refuse)
-    resp = await client.post("/api/system/backup/destination/test", json=_DEST)
-    assert resp.status_code == 200
-    assert await resp.get_json() == {
-        "ok": False,
-        "message": "Couldn't reach http://archives.example:8081: nothing is listening there.",
-    }
-
-
-@pytest.mark.asyncio
-async def test_the_upload_limit_saves_and_a_missing_one_keeps_it(client, _sealing_key):
-    resp = await client.patch(
-        "/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": 50},
-    )
-    assert (await resp.get_json())["upload_limit_mbps"] == 50
-    resp = await client.patch("/api/system/backup/destination", json=_DEST)
-    assert (await resp.get_json())["upload_limit_mbps"] == 50
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("limit", [-1, 2.5, "fast", True])
-async def test_an_upload_limit_that_isnt_a_whole_number_is_refused(client, _sealing_key, limit):
-    resp = await client.patch(
-        "/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": limit},
-    )
-    assert resp.status_code == 400
-    assert "upload_limit_mbps" in (await resp.get_json())["detail"]

tests/test_backup_media.py

@@ -1,264 +0,0 @@
-"""The off-site copy of the library's files (services/backup_media.py and the
-backup_media_task chunk loop): each image and attachment goes to the bucket
-once, by sha256, the work list is what isn't there yet, and a file that can't
-be sent is reported and tried again rather than stopping the run."""
-from datetime import UTC, datetime, timedelta
-
-import pytest
-from sqlalchemy import select
-
-from backend.app.models import (
-    Artist,
-    BackupBlob,
-    BackupRun,
-    ImageRecord,
-    Post,
-    PostAttachment,
-)
-from backend.app.services import backup_media
-from backend.app.services.backup_s3 import ListPage, S3Error, S3Target, StoredObject
-
-pytestmark = pytest.mark.integration
-
-TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK")
-TID = "http://archives.example:8081/curator"
-
-
-def _sha(c: str) -> str:
-    return c * 64
-
-
-def _image(db_sync, tmp_path, c, body=b"img", ext=".jpg"):
-    path = tmp_path / f"{c}{ext}"
-    path.write_bytes(body)
-    db_sync.add(ImageRecord(path=str(path), sha256=_sha(c), size_bytes=len(body),
-                            mime="image/jpeg", origin="imported_filesystem"))
-    return path
-
-
-def _attachment(db_sync, tmp_path, c, body=b"zip", name="set.zip"):
-    path = tmp_path / f"{c}-{name}"
-    path.write_bytes(body)
-    db_sync.add(PostAttachment(sha256=_sha(c), path=str(path), original_filename=name,
-                               ext=".zip", size_bytes=len(body)))
-    return path
-
-
-# ── keys and the work list ──────────────────────────────────────────────────
-
-
-def test_keys_are_content_addressed_under_media():
-    sha = "abcd" + "0" * 60
-    assert backup_media.blob_key(sha, "/images/x/Pic.JPG") == f"media/ab/cd/{sha}.jpg"
-    assert backup_media.blob_key(sha, "/images/x/noext") == f"media/ab/cd/{sha}"
-    assert backup_media.blob_key(sha, "/images/x/a.we ird") == f"media/ab/cd/{sha}"
-
-
-def test_the_work_list_is_every_file_once_that_the_bucket_lacks(db_sync, tmp_path):
-    _image(db_sync, tmp_path, "a")
-    _image(db_sync, tmp_path, "b")
-    _attachment(db_sync, tmp_path, "c")
-    # The same archive held by a post as well is one file to send.
-    artist = Artist(name="Bea", slug="bea")
-    db_sync.add(artist)
-    db_sync.flush()
-    post = Post(artist_id=artist.id, external_post_id="p1")
-    db_sync.add(post)
-    db_sync.flush()
-    db_sync.add(PostAttachment(post_id=post.id, sha256=_sha("c"),
-                               path=str(tmp_path / "c-copy.zip"),
-                               original_filename="set.zip", ext=".zip", size_bytes=3))
-    db_sync.add(BackupBlob(target=TID, sha256=_sha("b"), key="media/bb", size_bytes=3))
-    # A row for another bucket doesn't count for this one.
-    db_sync.add(BackupBlob(target="http://other/bucket", sha256=_sha("a"), key="k",
-                           size_bytes=3))
-    db_sync.commit()
-
-    shas = [b.sha256 for b in backup_media.pending(db_sync, TID)]
-    assert shas == [_sha("a"), _sha("c")]
-    assert [b.sha256 for b in backup_media.pending(db_sync, TID, after=_sha("a"))] == [_sha("c")]
-
-    cov = backup_media.coverage(db_sync, TID)
-    assert cov == {"files_total": 3, "bytes_total": 9, "files_sent": 1, "bytes_sent": 3}
-
-
-# ── sending one file ────────────────────────────────────────────────────────
-
-
-@pytest.fixture
-def puts(monkeypatch):
-    sent = []
-    monkeypatch.setattr(backup_media, "put_file",
-                        lambda t, key, path, **kw: sent.append(("whole", key, kw)))
-    monkeypatch.setattr(backup_media, "put_file_in_parts",
-                        lambda t, key, path, **kw: sent.append(("parts", key, kw)))
-    return sent
-
-
-def test_a_file_is_signed_with_its_recorded_sha(db_sync, tmp_path, puts):
-    path = _image(db_sync, tmp_path, "a")
-    blob = backup_media.Blob(_sha("a"), str(path), 3, "image/jpeg")
-    key = backup_media.send(TARGET, blob, tmp_path, None)
-    assert key == f"media/aa/aa/{_sha('a')}.jpg"
-    (how, sent_key, kw) = puts[0]
-    assert (how, sent_key) == ("whole", key)
-    assert kw["payload_sha256"] == _sha("a") and kw["content_type"] == "image/jpeg"
-
-
-def test_a_large_file_goes_in_parts(db_sync, tmp_path, puts, monkeypatch):
-    monkeypatch.setattr(backup_media, "MULTIPART_OVER", 2)
-    path = _attachment(db_sync, tmp_path, "c")
-    backup_media.send(TARGET, backup_media.Blob(_sha("c"), str(path), 3, None), tmp_path, None)
-    assert puts[0][0] == "parts"
-    assert puts[0][2]["sha256"] == _sha("c")
-
-
-def test_a_missing_or_changed_file_is_refused_before_sending(db_sync, tmp_path, puts):
-    gone = backup_media.Blob(_sha("a"), str(tmp_path / "gone.jpg"), 3, None)
-    with pytest.raises(S3Error, match="not on disk"):
-        backup_media.send(TARGET, gone, tmp_path, None)
-    path = _image(db_sync, tmp_path, "b", body=b"longer now")
-    changed = backup_media.Blob(_sha("b"), str(path), 3, None)
-    with pytest.raises(S3Error, match="when imported"):
-        backup_media.send(TARGET, changed, tmp_path, None)
-    assert puts == []
-
-
-def test_a_refusal_for_an_object_already_there_counts_as_sent(db_sync, tmp_path, monkeypatch):
-    def refused(*a, **kw):
-        raise S3Error("HTTP 403 AccessDenied: append-only.")
-    monkeypatch.setattr(backup_media, "put_file", refused)
-    monkeypatch.setattr(backup_media, "object_present", lambda t, key, size: True)
-    path = _image(db_sync, tmp_path, "a")
-    key = backup_media.send(TARGET, backup_media.Blob(_sha("a"), str(path), 3, None),
-                            tmp_path, None)
-    assert key.startswith("media/aa/aa/")
-
-
-# ── the run ─────────────────────────────────────────────────────────────────
-
-
-@pytest.fixture
-def run_env(monkeypatch, tmp_path):
-    monkeypatch.setattr("backend.app.tasks.backup.IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(
-        "backend.app.services.backup_destination.saved_target", lambda settings: TARGET,
-    )
-    queued = []
-    from backend.app.tasks import backup as tasks
-    monkeypatch.setattr(tasks.backup_media_task, "delay", lambda **kw: queued.append(kw))
-    return queued
-
-
-def _send_all_but(bad_sha):
-    def send(target, blob, images_root, pacer):
-        if blob.sha256 == bad_sha:
-            raise S3Error("HTTP 503: busy.")
-        return backup_media.blob_key(blob.sha256, blob.path)
-    return send
-
-
-def test_a_run_sends_what_is_missing_and_reports_what_failed(
-    db_sync, tmp_path, run_env, monkeypatch,
-):
-    from backend.app.tasks.backup import backup_media_task
-    for c in "abc":
-        _image(db_sync, tmp_path, c)
-    db_sync.commit()
-    monkeypatch.setattr(backup_media, "send", _send_all_but(_sha("b")))
-
-    result = backup_media_task(triggered_by="manual")
-    assert result["continues"] is False
-    row = db_sync.get(BackupRun, result["backup_run_id"])
-    db_sync.refresh(row)
-    assert (row.kind, row.status, row.size_bytes) == ("media", "ok", 6)
-    assert row.manifest["sent"] == 2 and row.manifest["failed"] == 1
-    assert row.manifest["offsite"]["bucket"] == "curator"
-    assert "1 file(s) weren't sent" in row.error
-    assert row.manifest["failures"][0]["error"] == "HTTP 503: busy."
-    sent = set(db_sync.execute(select(BackupBlob.sha256)).scalars())
-    assert sent == {_sha("a"), _sha("c")}
-
-    # The next run tries only the file that failed.
-    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
-    again = backup_media_task(triggered_by="manual")
-    row = db_sync.get(BackupRun, again["backup_run_id"])
-    db_sync.refresh(row)
-    assert row.manifest["sent"] == 1 and row.error is None
-
-
-def test_a_run_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch):
-    from backend.app.tasks import backup as tasks
-    for c in "ab":
-        _image(db_sync, tmp_path, c)
-    db_sync.commit()
-    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
-    monkeypatch.setattr(tasks, "MEDIA_CHUNK_SECONDS", -1)
-
-    result = tasks.backup_media_task(triggered_by="manual")
-    assert result["continues"] is True
-    assert run_env == [{"run_id": result["backup_run_id"]}]
-    row = db_sync.get(BackupRun, result["backup_run_id"])
-    db_sync.refresh(row)
-    assert row.status == "running" and row.last_progress_at is not None
-
-
-def test_a_second_run_waits_for_the_one_going(db_sync, run_env):
-    from backend.app.tasks.backup import backup_media_task
-    now = datetime.now(UTC)
-    db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly",
-                          started_at=now, manifest={}))
-    db_sync.commit()
-    assert "already running" in backup_media_task(triggered_by="manual")["skipped"]
-
-
-def test_without_a_destination_the_run_fails_with_why(db_sync, monkeypatch):
-    from backend.app.tasks.backup import backup_media_task
-    monkeypatch.setattr(
-        "backend.app.services.backup_destination.saved_target", lambda settings: None,
-    )
-    with pytest.raises(S3Error):
-        backup_media_task(triggered_by="manual")
-    row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "media")).scalar_one()
-    assert row.status == "error" and "No off-site destination" in row.error
-
-
-def test_a_long_media_run_is_judged_stalled_by_its_heartbeat(db_sync):
-    from backend.app.tasks.maintenance import recover_stalled_backup_runs
-    long_ago = datetime.now(UTC) - timedelta(hours=20)
-    alive = BackupRun(kind="media", status="running", triggered_by="nightly",
-                      started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={})
-    silent = BackupRun(kind="media", status="running", triggered_by="nightly",
-                       started_at=long_ago, last_progress_at=long_ago, manifest={})
-    db_sync.add_all([alive, silent])
-    db_sync.commit()
-    assert recover_stalled_backup_runs() == 1
-    db_sync.refresh(alive)
-    db_sync.refresh(silent)
-    assert (alive.status, silent.status) == ("running", "error")
-
-
-# ── reconcile ───────────────────────────────────────────────────────────────
-
-
-def test_reconcile_resends_what_the_bucket_lost_and_adopts_what_it_has(
-    db_sync, tmp_path, monkeypatch,
-):
-    _image(db_sync, tmp_path, "a")            # row, but the bucket lost it
-    _image(db_sync, tmp_path, "b")            # in the bucket, no row
-    _image(db_sync, tmp_path, "c")            # neither
-    key_a = backup_media.blob_key(_sha("a"), "x.jpg")
-    key_b = backup_media.blob_key(_sha("b"), "x.jpg")
-    db_sync.add(BackupBlob(target=TID, sha256=_sha("a"), key=key_a, size_bytes=3))
-    db_sync.commit()
-    pages = iter([
-        ListPage([StoredObject("media/zz/other", 1, "")], "next"),
-        ListPage([StoredObject(key_b, 3, "")], None),
-    ])
-    monkeypatch.setattr(backup_media, "list_objects", lambda *a, **kw: next(pages))
-    monkeypatch.setattr(backup_media.time, "sleep", lambda s: None)
-
-    assert backup_media.reconcile(db_sync, TARGET) == {
-        "listed": 2, "dropped": 1, "adopted": 1,
-    }
-    assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}

tests/test_backup_s3.py

@@ -1,363 +0,0 @@
-"""The backup destination's S3 client (services/backup_s3.py): signatures match
-AWS's published Signature Version 4 examples, and the calls send and read what
-S3 expects, over a real HTTP connection to a local stand-in server."""
-import hashlib
-import socket
-import threading
-from datetime import UTC, datetime
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-
-import pytest
-
-from backend.app.services import backup_s3 as s3
-
-# AWS's example credentials and moment ("Signature Calculations for the
-# Authorization Header", Amazon S3 API reference).
-KEY_ID = "AKIAIOSFODNN7EXAMPLE"
-SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
-AT = datetime(2013, 5, 24, tzinfo=UTC)
-
-
-def _signature(headers):
-    return headers["Authorization"].rsplit("Signature=", 1)[1]
-
-
-def test_aws_example_get_object():
-    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt",
-                {"Range": "bytes=0-9"}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
-    assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"]
-    assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h[
-        "Authorization"]
-    assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
-
-
-def test_aws_example_list_objects():
-    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
-                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
-    assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
-
-
-def test_a_changed_secret_changes_the_signature():
-    url = "https://examplebucket.s3.amazonaws.com/test.txt"
-    a = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
-    b = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT)
-    assert _signature(a) != _signature(b)
-
-
-def test_paths_are_path_style_and_keys_are_escaped():
-    t = s3.S3Target("http://archives.example:8081/s3", "curator", KEY_ID, SECRET)
-    assert t.url("media/ab/cd/abcd.jpg") == "http://archives.example:8081/s3/curator/media/ab/cd/abcd.jpg"
-    assert t.url() == "http://archives.example:8081/s3/curator/"
-    assert t.url("a b+c.jpg") == "http://archives.example:8081/s3/curator/a%20b%2Bc.jpg"
-
-
-# ── Calls, against a local stand-in ─────────────────────────────────────────
-
-
-class _Server:
-    """A local HTTP server that records each request and answers with
-    `reply(handler) -> (status, body)` or `(status, body, headers)`."""
-
-    def __init__(self, reply):
-        self.seen = []
-        outer = self
-
-        class Handler(BaseHTTPRequestHandler):
-            def _serve(self):
-                length = int(self.headers.get("Content-Length") or 0)
-                outer.seen.append({
-                    "method": self.command, "path": self.path,
-                    "headers": {k.lower(): v for k, v in self.headers.items()},
-                    "body": self.rfile.read(length) if length else b"",
-                })
-                status, body, *extra = reply(self)
-                self.send_response(status)
-                for name, value in (extra[0] if extra else {}).items():
-                    self.send_header(name, value)
-                self.send_header("Content-Length", str(len(body)))
-                self.end_headers()
-                self.wfile.write(body)
-
-            do_GET = do_PUT = do_POST = do_DELETE = _serve
-
-            def log_message(self, *args):
-                pass
-
-        self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
-        self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
-
-    def __enter__(self):
-        self.thread.start()
-        return self
-
-    def __exit__(self, *exc):
-        self.httpd.shutdown()
-        self.httpd.server_close()
-
-    def target(self, bucket="curator"):
-        host, port = self.httpd.server_address
-        return s3.S3Target(f"http://{host}:{port}", bucket, KEY_ID, SECRET)
-
-
-def test_put_streams_the_file_signed(tmp_path):
-    f = tmp_path / "abcd.jpg"
-    f.write_bytes(b"x" * 5000)
-    with _Server(lambda h: (200, b"")) as srv:
-        s3.put_file(srv.target(), "media/ab/cd/abcd.jpg", f, content_type="image/jpeg")
-    req = srv.seen[0]
-    assert req["method"] == "PUT"
-    assert req["path"] == "/curator/media/ab/cd/abcd.jpg"
-    assert req["body"] == b"x" * 5000
-    assert req["headers"]["x-amz-content-sha256"] == s3.file_sha256(f)
-    assert req["headers"]["content-type"] == "image/jpeg"
-    assert req["headers"]["authorization"].startswith(
-        "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/")
-
-
-def test_put_signs_with_a_known_sha_without_rereading(tmp_path, monkeypatch):
-    f = tmp_path / "b.bin"
-    f.write_bytes(b"y")
-    monkeypatch.setattr(s3, "file_sha256", lambda p: pytest.fail("read the file twice"))
-    with _Server(lambda h: (200, b"")) as srv:
-        s3.put_file(srv.target(), "b.bin", f, payload_sha256="a" * 64)
-    assert srv.seen[0]["headers"]["x-amz-content-sha256"] == "a" * 64
-
-
-def test_a_refusal_says_why(tmp_path):
-    f = tmp_path / "b.bin"
-    f.write_bytes(b"x")
-    body = b"<Error><Code>SignatureDoesNotMatch</Code><Message>no</Message></Error>"
-    with _Server(lambda h: (403, body)) as srv, pytest.raises(
-        s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong",
-    ):
-        s3.put_file(srv.target(), "b.bin", f)
-
-
-def test_a_web_server_at_the_address_is_named_as_not_s3():
-    """An address without the S3 port lands on whatever serves the default
-    port, and its plain 404 says nothing about S3."""
-    page = b"<html><body><h1>404 Not Found</h1></body></html>"
-    with _Server(lambda h: (404, page)) as srv, pytest.raises(
-        s3.S3Error,
-        match=r"HTTP 404: the address answered, but not as S3 — check the scheme and the port",
-    ):
-        s3.list_objects(srv.target())
-
-
-def test_a_server_error_without_a_body_is_not_blamed_on_the_port():
-    with _Server(lambda h: (503, b"")) as srv, pytest.raises(
-        s3.S3Error, match="HTTP 503: Service Unavailable",
-    ) as err:
-        s3.list_objects(srv.target())
-    assert "port" not in str(err.value)
-
-
-def test_nothing_listening_says_check_the_port():
-    with socket.socket() as sock:
-        sock.bind(("127.0.0.1", 0))
-        port = sock.getsockname()[1]
-    target = s3.S3Target(f"http://127.0.0.1:{port}", "curator", KEY_ID, SECRET)
-    with pytest.raises(s3.S3Error, match="nothing is listening there"):
-        s3.list_objects(target)
-
-
-def test_list_reads_contents_and_the_next_page_token():
-    xml = b"""<?xml version="1.0" encoding="UTF-8"?>
-<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-  <Name>curator</Name><KeyCount>2</KeyCount><IsTruncated>true</IsTruncated>
-  <NextContinuationToken>tok-2</NextContinuationToken>
-  <Contents><Key>database/fabledcurator-20261105T030000Z.dump</Key><Size>1200</Size>
-    <LastModified>2026-11-05T03:00:01.000Z</LastModified></Contents>
-  <Contents><Key>database/fabledcurator-20261106T030000Z.dump</Key><Size>1300</Size>
-    <LastModified>2026-11-06T03:00:01.000Z</LastModified></Contents>
-</ListBucketResult>"""
-    with _Server(lambda h: (200, xml)) as srv:
-        page = s3.list_objects(srv.target(), "database/", continuation="tok-1")
-    path = srv.seen[0]["path"]
-    assert path.startswith("/curator/?")
-    assert "list-type=2" in path and "prefix=database%2F" in path
-    assert "continuation-token=tok-1" in path
-    assert page.next_token == "tok-2"
-    assert page.objects == [
-        s3.StoredObject("database/fabledcurator-20261105T030000Z.dump", 1200,
-                        "2026-11-05T03:00:01.000Z"),
-        s3.StoredObject("database/fabledcurator-20261106T030000Z.dump", 1300,
-                        "2026-11-06T03:00:01.000Z"),
-    ]
-
-
-def test_a_listing_that_isnt_xml_is_named():
-    with _Server(lambda h: (200, b"hello")) as srv, pytest.raises(
-        s3.S3Error, match="isn't S3's XML",
-    ):
-        s3.list_objects(srv.target())
-
-
-def test_get_streams_the_object_to_disk(tmp_path):
-    with _Server(lambda h: (200, b"y" * 3000)) as srv:
-        out = tmp_path / "b.dump"
-        s3.get_file(srv.target(), "database/b.dump", out)
-    assert srv.seen[0]["path"] == "/curator/database/b.dump"
-    assert srv.seen[0]["headers"]["x-amz-content-sha256"] == s3.EMPTY_SHA256
-    assert out.read_bytes() == b"y" * 3000
-    assert not (tmp_path / "b.dump.partial").exists()
-
-
-def test_a_refused_get_says_so_and_writes_nothing(tmp_path):
-    body = b"<Error><Code>AccessDenied</Code></Error>"
-    out = tmp_path / "b.dump"
-    with _Server(lambda h: (403, body)) as srv, pytest.raises(s3.S3Error, match="b.dump"):
-        s3.get_file(srv.target(), "b.dump", out)
-    assert not out.exists()
-    assert not (tmp_path / "b.dump.partial").exists()
-
-
-# ── Multipart ───────────────────────────────────────────────────────────────
-
-_STARTED = b"<InitiateMultipartUploadResult><UploadId>up1</UploadId></InitiateMultipartUploadResult>"
-
-
-def _bucket(parts_reply=None, complete=(200, b"<CompleteMultipartUploadResult/>")):
-    """A stand-in bucket for one multipart upload: starts it, takes parts (each
-    answered by `parts_reply(n)`, an ETag by default) and completes it."""
-    def reply(h):
-        if h.command == "POST" and h.path.endswith("?uploads"):
-            return 200, _STARTED
-        if h.command == "PUT":
-            n = int(h.path.split("partNumber=")[1].split("&")[0])
-            return parts_reply(n) if parts_reply else (200, b"", {"ETag": f'"e{n}"'})
-        if h.command == "POST":
-            return complete
-        return 204, b""
-    return reply
-
-
-def test_an_upload_sends_its_parts_then_joins_them_in_order():
-    with _Server(_bucket()) as srv:
-        up = s3.Upload(srv.target(), "database/x.dump")
-        up.add(b"first,")
-        up.add(b"second")
-        up.complete()
-    start, one, two, done = srv.seen
-    assert (start["method"], start["path"]) == ("POST", "/curator/database/x.dump?uploads")
-    assert one["path"] == "/curator/database/x.dump?partNumber=1&uploadId=up1"
-    assert two["body"] == b"second"
-    assert two["headers"]["x-amz-content-sha256"] == hashlib.sha256(b"second").hexdigest()
-    assert done["path"] == "/curator/database/x.dump?uploadId=up1"
-    assert done["body"].index(b'"e1"') < done["body"].index(b'"e2"')
-    assert (up.parts, up.size) == (2, len(b"first,second"))
-
-
-def test_a_refused_start_says_why():
-    with _Server(lambda h: (403, b"<Error><Code>AccessDenied</Code></Error>")) as srv:
-        with pytest.raises(s3.S3Error, match="the key isn't allowed"):
-            s3.Upload(srv.target(), "database/x.dump")
-
-
-def test_a_completion_refused_under_a_200_is_a_failure():
-    error = (200, b"<Error><Code>InternalError</Code><Message>disk full</Message></Error>")
-    with _Server(_bucket(complete=error)) as srv:
-        up = s3.Upload(srv.target(), "database/x.dump")
-        up.add(b"data")
-        with pytest.raises(s3.S3Error, match="Finishing the upload"):
-            up.complete()
-
-
-def test_nothing_sent_is_not_completed():
-    with _Server(_bucket()) as srv:
-        up = s3.Upload(srv.target(), "database/x.dump")
-        with pytest.raises(s3.S3Error, match="Nothing was sent"):
-            up.complete()
-
-
-def test_a_busy_bucket_gets_the_part_again_after_a_rest(monkeypatch):
-    rests = []
-    monkeypatch.setattr(s3.time, "sleep", rests.append)
-    answers = iter([(503, b"", {"Retry-After": "7"}), (200, b"", {"ETag": '"e1"'})])
-    with _Server(_bucket(parts_reply=lambda n: next(answers))) as srv:
-        up = s3.Upload(srv.target(), "database/x.dump")
-        up.add(b"data")
-    assert rests == [7]
-    assert up.parts == 1
-
-
-def test_a_part_refused_outright_is_not_sent_again(monkeypatch):
-    monkeypatch.setattr(s3.time, "sleep", lambda s: pytest.fail("rested on a refusal"))
-    refused = (403, b"<Error><Code>AccessDenied</Code></Error>")
-    with _Server(_bucket(parts_reply=lambda n: refused)) as srv:
-        up = s3.Upload(srv.target(), "database/x.dump")
-        with pytest.raises(s3.S3Error, match="Part 1"):
-            up.add(b"data")
-
-
-def test_a_part_that_never_lands_gives_up_after_the_rests(monkeypatch):
-    rests = []
-    monkeypatch.setattr(s3.time, "sleep", rests.append)
-    with _Server(_bucket(parts_reply=lambda n: (503, b""))) as srv:
-        up = s3.Upload(srv.target(), "database/x.dump")
-        with pytest.raises(s3.S3Error, match="Part 1"):
-            up.add(b"data")
-    assert rests == list(s3.PART_RETRY_WAITS)
-
-
-def test_abort_asks_the_bucket_to_drop_the_parts_and_never_raises():
-    with _Server(_bucket()) as srv:
-        up = s3.Upload(srv.target(), "database/x.dump")
-        up.abort()
-        assert srv.seen[-1]["method"] == "DELETE"
-        assert srv.seen[-1]["path"] == "/curator/database/x.dump?uploadId=up1"
-    up.abort()  # the server is gone now: still no exception
-
-
-def test_the_pacer_holds_bytes_to_the_limit(monkeypatch):
-    clock = [0.0]
-    monkeypatch.setattr(s3.time, "monotonic", lambda: clock[0])
-    monkeypatch.setattr(s3.time, "sleep", lambda secs: clock.__setitem__(0, clock[0] + secs))
-    pacer = s3.Pacer(8)                     # 8 Mbit/s = 1,000,000 bytes/s
-    data = b"x" * (3 * s3.CHUNK)
-    assert b"".join(pacer.chunks(data)) == data
-    assert clock[0] == pytest.approx(len(data) / 1_000_000)
-
-
-def test_no_limit_never_waits(monkeypatch):
-    monkeypatch.setattr(s3.time, "sleep", lambda secs: pytest.fail("waited without a limit"))
-    assert b"".join(s3.Pacer(0).chunks(b"y" * (2 * s3.CHUNK + 1))) == b"y" * (2 * s3.CHUNK + 1)
-
-
-def test_a_large_file_is_sent_in_parts_and_checked_against_its_sha(tmp_path, monkeypatch):
-    monkeypatch.setattr(s3, "PART_SIZE", 4)
-    f = tmp_path / "big.zip"
-    f.write_bytes(b"0123456789")
-    with _Server(_bucket()) as srv:
-        sent = s3.put_file_in_parts(srv.target(), "media/x.zip", f,
-                                    sha256=hashlib.sha256(b"0123456789").hexdigest())
-    assert sent == 10
-    assert [r["body"] for r in srv.seen if r["method"] == "PUT"] == [b"0123", b"4567", b"89"]
-    assert srv.seen[-1]["method"] == "POST"             # completed
-
-
-def test_a_file_that_changed_since_import_is_not_completed(tmp_path, monkeypatch):
-    monkeypatch.setattr(s3, "PART_SIZE", 4)
-    f = tmp_path / "big.zip"
-    f.write_bytes(b"0123456789")
-    with _Server(_bucket()) as srv:
-        with pytest.raises(s3.S3Error, match="recorded checksum"):
-            s3.put_file_in_parts(srv.target(), "media/x.zip", f, sha256="0" * 64)
-    assert srv.seen[-1]["method"] == "DELETE"           # aborted, nothing stored
-
-
-def test_a_paced_put_sends_the_same_bytes(tmp_path):
-    f = tmp_path / "a.bin"
-    f.write_bytes(b"z" * (s3.CHUNK + 5))
-    with _Server(lambda h: (200, b"")) as srv:
-        s3.put_file(srv.target(), "a.bin", f, pacer=s3.Pacer(0))
-    assert srv.seen[0]["body"] == b"z" * (s3.CHUNK + 5)
-
-
-def test_presence_is_the_exact_key_at_the_exact_size():
-    listing = (b"<ListBucketResult><Contents><Key>media/a.jpg</Key><Size>3</Size>"
-               b"</Contents></ListBucketResult>")
-    with _Server(lambda h: (200, listing)) as srv:
-        assert s3.object_present(srv.target(), "media/a.jpg", 3)
-        assert not s3.object_present(srv.target(), "media/a.jpg", 4)
-        assert not s3.object_present(srv.target(), "media/a.jp", 3)

tests/test_backup_service.py

@@ -5,7 +5,6 @@ without external binaries. The real subprocess behavior is exercised
 implicitly via the Celery task tests in test_tasks_backup.py.
 """
 import json
-import re
 import subprocess
 from pathlib import Path
 
@@ -29,7 +28,7 @@ def fake_subprocess(monkeypatch):
         stderr = b""
 
     def _write_sentinel(cmd):
-        if "pg_dump" in cmd:
+        if cmd[0] == "pg_dump":
             i = cmd.index("-f")
             Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
         elif cmd[0] == "tar" and "-cf" in cmd:
@@ -88,7 +87,7 @@ def test_backup_db_strips_sqlalchemy_psycopg_driver(tmp_path, fake_subprocess):
         db_url="postgresql+psycopg://u:p@h/d", images_root=tmp_path,
     )
     cmd = fake_subprocess[0]
-    assert "pg_dump" in cmd
+    assert cmd[0] == "pg_dump"
     assert cmd[-1].startswith("postgresql://")
     assert "+psycopg" not in cmd[-1]
 
@@ -285,132 +284,3 @@ def test_run_bounded_fails_fast_when_unkillable(monkeypatch):
     with pytest.raises(subprocess.TimeoutExpired):
         backup_service._run_bounded(["pg_dump"], 1)
     assert killed["n"] == 1
-
-
-# --- backup_db_offsite: pg_dump streamed to the bucket ---------------
-
-
-class _Dump:
-    """A pg_dump stand-in: `output` on stdout, `said` on stderr, exiting with
-    `code` once its output has been read."""
-
-    def __init__(self, output=b"", said=b"", code=0):
-        self.output, self.said, self.code = output, said, code
-        self.cmd = None
-        self.killed = False
-
-    def __call__(self, cmd, stdout=None, stderr=None, **kwargs):
-        import io
-        self.cmd = list(cmd)
-        self.args = cmd
-        self.stdout = io.BytesIO(self.output)
-        stderr.write(self.said)
-        self.returncode = None
-        return self
-
-    def poll(self):
-        return self.returncode
-
-    def wait(self, timeout=None):
-        if self.returncode is None:
-            self.returncode = -9 if self.killed else self.code
-        return self.returncode
-
-    def kill(self):
-        self.killed = True
-
-
-class _Bucket:
-    """Records what an Upload was asked to do instead of sending it."""
-
-    def __init__(self, fail_on_part=None):
-        self.fail_on_part = fail_on_part
-        self.uploads = []
-
-    def __call__(self, target, key, **kwargs):
-        from backend.app.services.backup_s3 import S3Error
-        bucket = self
-
-        class _Upload:
-            def __init__(self):
-                self.key, self.received = key, []
-                self.completed = self.aborted = False
-                bucket.uploads.append(self)
-
-            @property
-            def parts(self):
-                return len(self.received)
-
-            @property
-            def size(self):
-                return sum(len(p) for p in self.received)
-
-            def add(self, data):
-                if bucket.fail_on_part == len(self.received) + 1:
-                    raise S3Error("HTTP 503: busy.")
-                self.received.append(data)
-
-            def complete(self):
-                self.completed = True
-
-            def abort(self):
-                self.aborted = True
-
-        return _Upload()
-
-
-@pytest.fixture
-def offsite(monkeypatch):
-    from backend.app.services import backup_s3
-    monkeypatch.setattr(backup_s3, "PART_SIZE", 4)
-    bucket = _Bucket()
-    monkeypatch.setattr(backup_s3, "Upload", bucket)
-    target = backup_s3.S3Target("http://archives.example:8081", "curator", "AK", "SK")
-    return bucket, target
-
-
-def test_a_dump_streams_to_the_bucket_in_parts_with_nothing_kept_here(offsite, monkeypatch):
-    bucket, target = offsite
-    dump = _Dump(output=b"0123456789")
-    monkeypatch.setattr("subprocess.Popen", dump)
-    result = backup_service.backup_db_offsite(
-        db_url="postgresql+psycopg://u:p@h/d", target=target, limit_mbps=50,
-    )
-    (upload,) = bucket.uploads
-    assert upload.received == [b"0123", b"4567", b"89"]
-    assert upload.completed and not upload.aborted
-    assert re.fullmatch(r"database/fabledcurator-\d{8}T\d{6}Z\.dump", result["key"])
-    assert result["key"] == upload.key
-    assert (result["bucket"], result["size_bytes"], result["parts"]) == ("curator", 10, 3)
-    assert dump.cmd[0] == "nice" and "pg_dump" in dump.cmd and "-Fc" in dump.cmd
-    assert "-f" not in dump.cmd                  # stdout, not a file
-    assert dump.cmd[-1] == "postgresql://u:p@h/d"
-
-
-def test_a_failed_dump_leaves_nothing_in_the_bucket_and_says_why(offsite, monkeypatch):
-    bucket, target = offsite
-    said = b"pg_dump: dumping contents of table x\npg_dump: error: connection lost\n"
-    monkeypatch.setattr("subprocess.Popen", _Dump(output=b"012345", said=said, code=1))
-    with pytest.raises(backup_service.DumpFailed, match="^pg_dump: error: connection lost$"):
-        backup_service.backup_db_offsite(db_url="postgresql://u@h/d", target=target)
-    (upload,) = bucket.uploads
-    assert upload.aborted and not upload.completed
-
-
-def test_a_bucket_that_stops_taking_parts_stops_the_dump(offsite, monkeypatch):
-    from backend.app.services.backup_s3 import S3Error
-    bucket, target = offsite
-    bucket.fail_on_part = 2
-    dump = _Dump(output=b"0123456789")
-    monkeypatch.setattr("subprocess.Popen", dump)
-    with pytest.raises(S3Error):
-        backup_service.backup_db_offsite(db_url="postgresql://u@h/d", target=target)
-    assert dump.killed
-    (upload,) = bucket.uploads
-    assert upload.aborted and not upload.completed
-
-
-def test_dump_names_sort_by_time():
-    assert backup_service.offsite_dump_key("20261107T030000Z") == (
-        "database/fabledcurator-20261107T030000Z.dump"
-    )

tests/test_download_service.py

@@ -193,69 +193,6 @@ async def test_download_source_attaches_written_files(
     assert imported_when_marked == [2]
 
 
-@pytest.mark.asyncio
-async def test_import_step_keeps_only_the_files_the_library_needs_on_disk(
-    db, db_sync, tmp_path, seed_artist_and_source,
-):
-    """After the import step, a downloaded image the library took stays where it
-    is, while an archive whose images were extracted (the archive is in the
-    attachments store) and an image the import filters rejected are removed, so
-    nothing is left on disk without a record for a later walk to import again."""
-    import zipfile
-
-    from PIL import Image
-
-    from backend.app.models import PostAttachment
-    from backend.app.services.download_service import DownloadService
-    from backend.app.services.importer import Importer
-
-    _artist, source = seed_artist_and_source
-    images_root = tmp_path / "images"
-    post_dir = images_root / "alice" / "patreon" / "post"
-
-    kept = post_dir / "a.jpg"
-    _make_jpg(kept, split="h")
-    member = tmp_path / "member.jpg"
-    _make_jpg(member, split="v")
-    archive = post_dir / "pack.zip"
-    with zipfile.ZipFile(archive, "w") as z:
-        z.writestr("art.jpg", member.read_bytes())
-    tiny = post_dir / "tiny.png"
-    Image.new("RGB", (16, 16), (200, 30, 30)).save(tiny)  # below min_width
-
-    result = _make_fake_dl_result(
-        success=True, written_paths=[str(kept), str(archive), str(tiny)],
-        files_downloaded=3,
-    )
-    sync_settings = db_sync.execute(
-        select(ImportSettings).where(ImportSettings.id == 1)
-    ).scalar_one()
-    sync_settings.phash_threshold = 0
-    sync_settings.min_width = 32
-    importer = Importer(
-        session=db_sync, images_root=images_root, import_root=images_root,
-        thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
-    )
-    crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
-    svc = DownloadService(
-        async_session=db, sync_session=db_sync, gdl=_fake_gdl_with_result(result),
-        importer=importer, cred_service=CredentialService(db, crypto),
-    )
-    _stub_patreon_ingester(svc, result)
-    await svc.download_source(source.id)
-
-    assert kept.exists()
-    assert not archive.exists()
-    assert not tiny.exists()
-    # The archive and its image are both still in the library.
-    assert db_sync.execute(
-        select(func.count(PostAttachment.id)).where(
-            PostAttachment.original_filename == "pack.zip"
-        )
-    ).scalar_one() == 1
-    assert db_sync.execute(select(func.count(ImageRecord.id))).scalar_one() == 2
-
-
 @pytest.mark.asyncio
 async def test_download_source_skipped_when_disabled(
     db, db_sync, tmp_path, seed_artist_and_source,

tests/test_external_fetch.py

@@ -108,27 +108,6 @@ def test_mega_collects_new_files(tmp_path, monkeypatch):
     assert res.files[0].name == "film.mp4"
 
 
-def test_mega_folder_link_collects_files_in_subfolders(tmp_path, monkeypatch):
-    """A folder link keeps the folder's tree; every file in it is collected,
-    files already there before the fetch and staging files are not."""
-    (tmp_path / "old.png").write_bytes(b"earlier")
-
-    def fake_mega(url, out_dir, *, timeout):
-        nested = Path(out_dir) / "images" / "2020"
-        nested.mkdir(parents=True)
-        (nested / "kara.png").write_bytes(b"png")
-        (Path(out_dir) / "videos").mkdir()
-        (Path(out_dir) / "videos" / "loop.webm").write_bytes(b"webm")
-        (nested / ".megatmp.AbCd").write_bytes(b"half")
-
-    monkeypatch.setattr(ef, "_run_mega_get", fake_mega)
-    res = fetch_external("mega", "https://mega.nz/folder/x#k", tmp_path)
-    assert res.ok
-    assert sorted(p.relative_to(tmp_path).as_posix() for p in res.files) == [
-        "images/2020/kara.png", "videos/loop.webm",
-    ]
-
-
 def test_unsupported_host(tmp_path):
     res = fetch_external("nope", "https://x", tmp_path)
     assert not res.ok

tests/test_external_worker.py

@@ -251,293 +251,3 @@ def test_downloaded_archive_gets_provenance_and_tagging(db_sync, tmp_path, monke
     # Tagging + thumbnails queued for exactly those member images.
     assert set(tagged) == member_ids
     assert set(thumbed) == member_ids
-
-
-def _link_dir(tmp_path, host, link_id):
-    return tmp_path / f"ext-{host}" / "patreon" / "external" / f"EXT-{host}" / str(link_id)
-
-
-def test_extracted_archive_is_not_left_in_the_link_dir(db_sync, tmp_path, monkeypatch):
-    """An archive whose images were extracted is in the attachments store, so
-    the downloaded copy is removed instead of sitting beside it."""
-    _, link = _seed(db_sync, host="mega")
-    link_id = link.id
-    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
-    written = []
-
-    def fake_fetch(host, url, dest_dir, **kwargs):
-        dest_dir.mkdir(parents=True, exist_ok=True)
-        zpath = dest_dir / "pack.zip"
-        with zipfile.ZipFile(zpath, "w") as z:
-            z.writestr("art.jpg", _structured_jpeg_bytes())
-        written.append(zpath)
-        return FetchResult(files=[zpath], bytes=zpath.stat().st_size)
-
-    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
-    import backend.app.tasks.ml as ml_mod
-    import backend.app.tasks.thumbnail as thumb_mod
-    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
-    monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
-
-    out = ext.fetch_external_link(link_id)
-    assert out["images"] >= 1
-    assert not written[0].exists()
-    db_sync.expire_all()
-    assert db_sync.execute(
-        select(func.count(PostAttachment.id))
-        .where(PostAttachment.original_filename == "pack.zip")
-    ).scalar_one() == 1
-
-
-def test_a_failed_fetch_leaves_no_partial_files(db_sync, tmp_path, monkeypatch):
-    """A fetch that fails midway (megatools leaves `.megatmp.*` behind) clears
-    the link's staging dir, so a dead link leaves nothing on disk."""
-    _, link = _seed(db_sync, host="mega")
-    link_id = link.id
-    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
-
-    def fake_fetch(host, url, dest_dir, **kwargs):
-        dest_dir.mkdir(parents=True, exist_ok=True)
-        (dest_dir / ".megatmp.AbCd").write_bytes(b"half a game build")
-        return FetchResult(error="megatools dl exit 1: timed out")
-
-    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
-    ext.fetch_external_link(link_id)
-
-    link_dir = _link_dir(tmp_path, "mega", link_id)
-    assert not (link_dir / ".megatmp.AbCd").exists()
-    db_sync.expire_all()
-    assert db_sync.get(ExternalLink, link_id).status == "failed"
-
-
-def test_a_retry_clears_what_a_killed_attempt_left(db_sync, tmp_path, monkeypatch):
-    """Leftovers from an attempt whose worker was killed are removed before the
-    next fetch starts; an image imported in place by an earlier attempt stays."""
-    _, link = _seed(db_sync, host="pixeldrain")
-    link_id = link.id
-    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
-    link_dir = _link_dir(tmp_path, "pixeldrain", link_id)
-    link_dir.mkdir(parents=True)
-    leftover = link_dir / "build.zip.part"
-    leftover.write_bytes(b"partial")
-    imported = link_dir / "kept.jpg"
-    imported.write_bytes(_structured_jpeg_bytes())
-    db_sync.add(ImageRecord(
-        path=str(imported), sha256="ext-kept".ljust(64, "x"),
-        size_bytes=imported.stat().st_size, mime="image/jpeg",
-        width=256, height=256, origin="downloaded", integrity_status="ok",
-    ))
-    db_sync.commit()
-    seen_at_fetch = []
-
-    def fake_fetch(host, url, dest_dir, **kwargs):
-        seen_at_fetch.extend(sorted(p.name for p in dest_dir.iterdir()))
-        return FetchResult(error="HTTP 503")
-
-    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
-    ext.fetch_external_link(link_id)
-
-    assert seen_at_fetch == ["kept.jpg"]
-    assert imported.exists()
-    assert not leftover.exists()
-
-
-def test_a_rejected_download_is_removed(db_sync, tmp_path, monkeypatch):
-    """A downloaded image the import filters reject has no record, so it is
-    removed rather than left in the link's dir."""
-    from backend.app.models import ImportSettings
-
-    _, link = _seed(db_sync, host="dropbox")
-    link_id = link.id
-    settings = db_sync.execute(
-        select(ImportSettings).where(ImportSettings.id == 1)
-    ).scalar_one()
-    settings.min_width = 512
-    db_sync.commit()
-    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
-    written = []
-
-    def fake_fetch(host, url, dest_dir, **kwargs):
-        dest_dir.mkdir(parents=True, exist_ok=True)
-        f = dest_dir / "small.jpg"  # 256px wide, below min_width
-        f.write_bytes(_structured_jpeg_bytes())
-        written.append(f)
-        return FetchResult(files=[f], bytes=f.stat().st_size)
-
-    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
-    out = ext.fetch_external_link(link_id)
-
-    assert out["images"] == 0
-    assert not written[0].exists()
-    db_sync.expire_all()
-    assert db_sync.get(ExternalLink, link_id).status == "downloaded"
-
-
-def _other_jpeg_bytes():
-    # A second structured image, unlike _structured_jpeg_bytes, so the two
-    # don't near-duplicate each other at import.
-    im = Image.new("RGB", (256, 256))
-    px = im.load()
-    for y in range(256):
-        for x in range(256):
-            px[x, y] = ((x * x) % 256, (y * 3) % 256, (x ^ y) % 256)
-    buf = io.BytesIO()
-    im.save(buf, "JPEG")
-    return buf.getvalue()
-
-
-def test_a_failed_fetch_keeps_complete_files(db_sync, tmp_path, monkeypatch):
-    """A failed fetch removes staging files only. A complete file may be the
-    only copy left once the host has dropped the upload."""
-    _, link = _seed(db_sync, host="mega")
-    link_id = link.id
-    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
-
-    def fake_fetch(host, url, dest_dir, **kwargs):
-        (dest_dir / "images").mkdir(parents=True, exist_ok=True)
-        (dest_dir / "images" / "done.jpg").write_bytes(_structured_jpeg_bytes())
-        (dest_dir / "images" / ".megatmp.AbCd").write_bytes(b"half")
-        return FetchResult(error="API call 'f' failed: ENOENT")
-
-    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
-    ext.fetch_external_link(link_id)
-
-    link_dir = _link_dir(tmp_path, "mega", link_id)
-    assert (link_dir / "images" / "done.jpg").exists()
-    assert not (link_dir / "images" / ".megatmp.AbCd").exists()
-
-
-def test_a_successful_fetch_imports_what_an_earlier_attempt_finished(
-    db_sync, tmp_path, monkeypatch,
-):
-    """Complete files an earlier attempt wrote, in subfolders too, are imported
-    with the new fetch's files instead of being deleted or left behind."""
-    _, link = _seed(db_sync, host="mega")
-    link_id = link.id
-    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
-    link_dir = _link_dir(tmp_path, "mega", link_id)
-    (link_dir / "pack").mkdir(parents=True)
-    earlier = link_dir / "pack" / "earlier.jpg"
-    earlier.write_bytes(_structured_jpeg_bytes())
-
-    def fake_fetch(host, url, dest_dir, **kwargs):
-        f = dest_dir / "new.jpg"
-        f.write_bytes(_other_jpeg_bytes())
-        return FetchResult(files=[f], bytes=f.stat().st_size)
-
-    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
-    import backend.app.tasks.ml as ml_mod
-    import backend.app.tasks.thumbnail as thumb_mod
-    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
-    monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
-
-    out = ext.fetch_external_link(link_id)
-
-    assert out["images"] == 2
-    db_sync.expire_all()
-    paths = set(db_sync.execute(select(ImageRecord.path)).scalars())
-    assert {str(earlier), str(link_dir / "new.jpg")} <= paths
-
-
-def test_sweep_recovers_a_link_whose_worker_died(db_sync, monkeypatch):
-    """A link 'downloading' past the task's hard limit (or with no claim time)
-    counts as a failed attempt and is retried; a fresh claim is left alone."""
-    from datetime import UTC, datetime, timedelta
-
-    _, stale = _seed(db_sync, host="mega", status="downloading")
-    _, unclaimed = _seed(db_sync, host="gdrive", status="downloading")
-    _, fresh = _seed(db_sync, host="dropbox", status="downloading")
-    now = datetime.now(UTC)
-    stale.started_at = now - timedelta(minutes=ext._STALL_MINUTES + 5)
-    fresh.started_at = now - timedelta(minutes=5)
-    db_sync.commit()
-    stale_id, unclaimed_id, fresh_id = stale.id, unclaimed.id, fresh.id
-
-    enqueued = []
-    monkeypatch.setattr(ext.fetch_external_link, "delay", lambda lid: enqueued.append(lid))
-    out = ext.sweep_external_links()
-
-    assert out["recovered"] == 2
-    assert set(enqueued) == {stale_id, unclaimed_id}
-    db_sync.expire_all()
-    for lid in (stale_id, unclaimed_id):
-        row = db_sync.get(ExternalLink, lid)
-        assert (row.status, row.attempts) == ("failed", 1)
-    assert db_sync.get(ExternalLink, fresh_id).status == "downloading"
-
-
-def test_recovery_gives_a_dead_link_one_more_attempt_at_growing_intervals(db_sync, monkeypatch):
-    """A dead link is retried once after 2**recoveries days, not given a fresh
-    three-attempt budget, so a link that always fails isn't downloaded daily."""
-    from datetime import UTC, datetime, timedelta
-
-    now = datetime.now(UTC)
-    _, due = _seed(db_sync, host="mega", status="dead")
-    _, waiting = _seed(db_sync, host="gdrive", status="dead")
-    _, spent = _seed(db_sync, host="dropbox", status="dead")
-    due.attempts = waiting.attempts = spent.attempts = ext.DEAD_LETTER_THRESHOLD
-    due.recoveries, due.completed_at = 2, now - timedelta(days=4, hours=1)
-    waiting.recoveries, waiting.completed_at = 2, now - timedelta(days=3)
-    spent.recoveries, spent.completed_at = ext._MAX_RECOVERIES, now - timedelta(days=90)
-    db_sync.commit()
-    due_id, waiting_id, spent_id = due.id, waiting.id, spent.id
-
-    enqueued = []
-    monkeypatch.setattr(ext.fetch_external_link, "delay", lambda lid: enqueued.append(lid))
-    out = ext.recover_external_links()
-
-    assert out["revived"] == 1
-    assert enqueued == [due_id]
-    db_sync.expire_all()
-    row = db_sync.get(ExternalLink, due_id)
-    assert (row.status, row.attempts, row.recoveries) == (
-        "failed", ext.DEAD_LETTER_THRESHOLD - 1, 3,
-    )
-    assert db_sync.get(ExternalLink, waiting_id).status == "dead"
-    assert db_sync.get(ExternalLink, spent_id).status == "dead"
-
-
-def test_a_recovered_link_that_fails_again_goes_straight_back_to_dead(
-    db_sync, tmp_path, monkeypatch,
-):
-    _, link = _seed(db_sync, host="mega", status="dead")
-    link.attempts = ext.DEAD_LETTER_THRESHOLD
-    db_sync.commit()
-    link_id = link.id
-    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
-    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
-    monkeypatch.setattr(ext, "fetch_external", lambda *a, **k: FetchResult(error="too big"))
-    monkeypatch.setattr(ext.fetch_external_link, "delay", ext.fetch_external_link)
-
-    ext.recover_external_links()
-
-    db_sync.expire_all()
-    row = db_sync.get(ExternalLink, link_id)
-    assert (row.status, row.recoveries) == ("dead", 1)
-
-
-def test_prune_removes_only_links_recovery_has_given_up_on(db_sync):
-    from datetime import UTC, datetime, timedelta
-
-    old = datetime.now(UTC) - timedelta(days=ext._RETENTION_DAYS + 1)
-    _, spent = _seed(db_sync, host="mega", status="dead")
-    _, recovering = _seed(db_sync, host="gdrive", status="dead")
-    _, recent = _seed(db_sync, host="dropbox", status="dead")
-    spent.recoveries, spent.completed_at = ext._MAX_RECOVERIES, old
-    recovering.recoveries, recovering.completed_at = 1, old
-    recent.recoveries = ext._MAX_RECOVERIES
-    recent.completed_at = datetime.now(UTC) - timedelta(days=1)
-    db_sync.commit()
-    spent_id, recovering_id, recent_id = spent.id, recovering.id, recent.id
-
-    assert ext.prune_external_links() == {"pruned": 1}
-    db_sync.expire_all()
-    assert db_sync.get(ExternalLink, spent_id) is None
-    assert db_sync.get(ExternalLink, recovering_id) is not None
-    assert db_sync.get(ExternalLink, recent_id) is not None

tests/test_importer_attach_in_place.py

@@ -410,31 +410,6 @@ def test_attach_in_place_invalid_image_returns_skipped(importer):
     assert result.skip_reason.value == "invalid_image"
 
 
-@pytest.mark.parametrize("name", ["film.mp4abc123_.part", "img.jpg.partial", ".megatmp.Xy12"])
-def test_an_unfinished_download_is_refused_and_left_on_disk(importer, db_sync, name):
-    """A staging file is a download still being written (or abandoned): it is
-    never stored as an attachment, and the importer never deletes it, so a
-    download in progress is not pulled out from under its writer."""
-    from backend.app.models import PostAttachment
-
-    artist = Artist(name="Hal", slug="hal")
-    db_sync.add(artist)
-    db_sync.flush()
-    staging = importer.images_root / "hal" / "patreon" / "post" / name
-    staging.parent.mkdir(parents=True, exist_ok=True)
-    staging.write_bytes(b"half of a video")
-
-    for result in (importer.attach_in_place(staging, artist=artist),
-                   importer.import_one(staging)):
-        assert result.status == "skipped"
-        assert "unfinished download" in result.error
-        assert importer.discard_after_attach(staging, result) is False
-    assert staging.exists()
-    assert db_sync.execute(
-        select(func.count()).where(PostAttachment.original_filename == name)
-    ).scalar() == 0
-
-
 def test_attach_in_place_non_media_routes_to_attachment(importer, db_sync):
     """FC-2d-iii dispatch parity for the download path. A non-media file
     (.pdf, .txt etc.) downloaded by gallery-dl must become a PostAttachment,

tests/test_importer_sha_race.py

@@ -1,202 +0,0 @@
-"""Two workers importing the same picture at once (services/importer.py).
-
-The importer looks a file's sha256 up before inserting its row. When two copies
-of one picture are imported side by side, both lookups can find nothing; the
-unique index on sha256 then refuses the second insert. The loser files its copy
-as a duplicate of the winner, links its post, and leaves the session usable,
-instead of failing the import with a dead transaction.
-
-The race is staged the way tests/test_importer_provenance_race.py stages its
-own: the loser's sha256 lookup is forced to miss once, as it would if it ran
-just before the winner's commit. The loser's near-duplicate cache is empty for
-the same reason (it was loaded before the winner's row existed).
-"""
-
-import io
-import zipfile
-from pathlib import Path
-
-import pytest
-from PIL import Image
-from sqlalchemy import func, select
-
-from backend.app.models import Artist, ImageRecord, ImportSettings, PostAttachment
-from backend.app.services.importer import Importer, SkipReason
-from backend.app.services.thumbnailer import Thumbnailer
-
-pytestmark = pytest.mark.integration
-
-
-@pytest.fixture
-def layout(tmp_path):
-    import_root = tmp_path / "import"
-    images_root = tmp_path / "images"
-    import_root.mkdir()
-    images_root.mkdir()
-    return import_root, images_root
-
-
-def _worker(db_sync, layout) -> Importer:
-    import_root, images_root = layout
-    settings = db_sync.execute(
-        select(ImportSettings).where(ImportSettings.id == 1)
-    ).scalar_one()
-    return Importer(
-        session=db_sync, images_root=images_root, import_root=import_root,
-        thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
-    )
-
-
-def _jpeg(path: Path) -> bytes:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    Image.new("RGB", (320, 240), (40, 90, 160)).save(path, "JPEG")
-    return path.read_bytes()
-
-
-def _copy(src: Path, dst: Path) -> Path:
-    dst.parent.mkdir(parents=True, exist_ok=True)
-    dst.write_bytes(src.read_bytes())
-    return dst
-
-
-def _lose_the_race(importer: Importer, monkeypatch) -> None:
-    """Make `importer` miss its first sha256 lookup and see no near-duplicates."""
-    real_execute = importer.session.execute
-    missed = [False]
-
-    class _Miss:
-        def scalar_one_or_none(self):
-            return None
-
-    def execute(stmt, *args, **kwargs):
-        sql = str(stmt)
-        if not missed[0] and "FROM image_record" in sql and "image_record.sha256 =" in sql:
-            missed[0] = True
-            return _Miss()
-        return real_execute(stmt, *args, **kwargs)
-
-    monkeypatch.setattr(importer.session, "execute", execute)
-    monkeypatch.setattr(importer, "_phash_candidates_cache", lambda: [])
-
-
-def _records(db_sync) -> int:
-    return db_sync.execute(select(func.count()).select_from(ImageRecord)).scalar_one()
-
-
-def test_a_copy_imported_alongside_the_original_is_filed_as_its_duplicate(
-    db_sync, layout, monkeypatch,
-):
-    import_root, images_root = layout
-    first = import_root / "Alice" / "set1" / "pic.jpg"
-    _jpeg(first)
-    second = _copy(first, import_root / "Alice" / "set2" / "pic.jpg")
-
-    winner = _worker(db_sync, layout).import_one(first)
-    assert winner.status == "imported"
-    kept = Path(db_sync.get(ImageRecord, winner.image_id).path)
-
-    loser = _worker(db_sync, layout)
-    _lose_the_race(loser, monkeypatch)
-    result = loser.import_one(second)
-
-    assert result.status == "skipped"
-    assert result.skip_reason == SkipReason.duplicate_hash
-    assert result.image_id == winner.image_id
-    monkeypatch.undo()
-    assert _records(db_sync) == 1
-    assert kept.is_file()
-    # The loser's copy in the library was a stray and is gone; only the
-    # winner's file remains.
-    library_files = [p for p in images_root.rglob("*.jpg") if p.is_file()]
-    assert library_files == [kept]
-
-
-def test_the_same_file_imported_twice_at_once_keeps_its_library_copy(
-    db_sync, layout, monkeypatch,
-):
-    """Both runs write the same library path, so the loser's copy IS the
-    winner's file and must not be deleted."""
-    import_root, _ = layout
-    src = import_root / "Alice" / "pic.jpg"
-    _jpeg(src)
-
-    winner = _worker(db_sync, layout).import_one(src)
-    kept = Path(db_sync.get(ImageRecord, winner.image_id).path)
-
-    loser = _worker(db_sync, layout)
-    _lose_the_race(loser, monkeypatch)
-    result = loser.import_one(src)
-
-    assert result.skip_reason == SkipReason.duplicate_hash
-    assert kept.is_file()
-
-
-def test_a_copy_attached_in_place_alongside_the_original_is_its_duplicate(
-    db_sync, layout, monkeypatch,
-):
-    _, images_root = layout
-    artist = Artist(name="Bea", slug="bea")
-    db_sync.add(artist)
-    db_sync.commit()
-    first = images_root / "bea" / "patreon" / "post1" / "pic.jpg"
-    _jpeg(first)
-    second = _copy(first, images_root / "bea" / "patreon" / "post2" / "pic.jpg")
-
-    winner = _worker(db_sync, layout).attach_in_place(first, artist=artist)
-    assert winner.status == "imported"
-
-    loser = _worker(db_sync, layout)
-    _lose_the_race(loser, monkeypatch)
-    result = loser.attach_in_place(second, artist=artist)
-
-    assert result.status == "skipped"
-    assert result.skip_reason == SkipReason.duplicate_hash
-    assert result.image_id == winner.image_id
-    monkeypatch.undo()
-    assert _records(db_sync) == 1
-    # Removing the loser's file is the caller's step (discard_after_attach).
-    assert first.is_file() and second.is_file()
-
-
-def test_an_archive_whose_member_breaks_the_transaction_is_still_stored(
-    db_sync, layout, monkeypatch,
-):
-    """A member whose import fails mid-flush leaves the transaction dead. The
-    member loop rolls it back, so the archive itself is still stored and the
-    import ends with a result instead of an exception."""
-    import_root, _ = layout
-    seed = _worker(db_sync, layout).import_one(_jpeg_at(import_root / "Cy" / "seed.jpg"))
-    seed_path = db_sync.get(ImageRecord, seed.image_id).path
-
-    arc = import_root / "Cy" / "set.zip"
-    with zipfile.ZipFile(arc, "w") as zf:
-        zf.writestr("a.jpg", _jpeg_bytes())
-
-    importer = _worker(db_sync, layout)
-
-    def breaks_the_transaction(*args, **kwargs):
-        importer.session.add(ImageRecord(
-            path=seed_path, sha256="f" * 64, size_bytes=1, mime="image/jpeg",
-            origin="imported_filesystem",
-        ))
-        importer.session.flush()  # refused: the path is taken
-
-    monkeypatch.setattr(importer, "_import_media", breaks_the_transaction)
-    result = importer.import_one(arc)
-
-    assert result.status == "attached"
-    stored = db_sync.execute(
-        select(func.count()).select_from(PostAttachment)
-    ).scalar_one()
-    assert stored == 1
-
-
-def _jpeg_at(path: Path) -> Path:
-    _jpeg(path)
-    return path
-
-
-def _jpeg_bytes() -> bytes:
-    buf = io.BytesIO()
-    Image.new("RGB", (64, 48), (200, 30, 30)).save(buf, "JPEG")
-    return buf.getvalue()

tests/test_library_scan.py

@@ -1,123 +0,0 @@
-"""Finding files in the artists' folders that the library has no record of, and
-importing them where they lie (services.library_scan + the `library` batch)."""
-
-import io
-import os
-import time
-
-import pytest
-from PIL import Image
-from sqlalchemy import select
-
-import backend.app.tasks.import_file as import_file
-from backend.app.models import Artist, ImageRecord, ImportSettings, ImportTask
-from backend.app.services import library_scan
-from backend.app.services.importer import Importer
-from backend.app.services.thumbnailer import Thumbnailer
-
-pytestmark = pytest.mark.integration
-
-
-def _jpeg(seed: int) -> bytes:
-    im = Image.new("RGB", (256, 256))
-    px = im.load()
-    for y in range(256):
-        for x in range(256):
-            px[x, y] = ((x * seed) % 256, (y * (seed + 1)) % 256, (x ^ y) % 256)
-    buf = io.BytesIO()
-    im.save(buf, "JPEG")
-    return buf.getvalue()
-
-
-def _write(path, data: bytes, *, age_days: float = 2):
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_bytes(data)
-    t = time.time() - age_days * 86400
-    os.utime(path, (t, t))
-    return path
-
-
-@pytest.fixture
-def layout(db_sync, tmp_path):
-    db_sync.add(Artist(name="Libscan Maewix", slug="libscan-maewix"))
-    db_sync.commit()
-    post = tmp_path / "Libscan-Maewix" / "patreon" / "2026-05-01_123_Pack"
-    files = {
-        "missing": _write(post / "01_art.jpg", _jpeg(1)),
-        "owned": _write(post / "02_owned.jpg", _jpeg(2)),
-        "sidecar": _write(post / "01_art.json", b'{"category": "patreon", "id": "123"}'),
-        "staging": _write(post / "03_big.zip.part", b"partial"),
-        "recent": _write(post / "04_new.jpg", _jpeg(4), age_days=0),
-        "not_media": _write(post / "05_game.exe", b"MZ"),
-        "no_artist": _write(tmp_path / "libscan-nobody" / "x" / "a.jpg", _jpeg(5)),
-        "store": _write(tmp_path / "thumbs" / "abc" / "f.jpg", _jpeg(6)),
-    }
-    db_sync.add(ImageRecord(
-        path=str(files["owned"]), sha256="libscan-owned".ljust(64, "x"),
-        size_bytes=files["owned"].stat().st_size, mime="image/jpeg",
-        width=256, height=256, origin="downloaded", integrity_status="ok",
-    ))
-    db_sync.commit()
-    return tmp_path, files
-
-
-def test_finds_only_unrecorded_media_in_an_artists_folder(db_sync, layout):
-    """A differently-cased folder still matches its artist. Recorded files,
-    sidecars, staging files, the library's own stores, folders that match no
-    artist, files changed in the last day and non-media are not listed."""
-    root, files = layout
-    found, summary = library_scan.find_unimported(db_sync, root)
-
-    assert [u.path for u in found] == [files["missing"]]
-    assert found[0].artist_slug == "libscan-maewix"
-    assert found[0].task_type == "media"
-    assert summary["files"] == 1
-    assert summary["recent"] == 1
-    assert summary["not_media"] == 1
-    assert summary["no_artist"] == 1
-    assert summary["artists"][0]["slug"] == "libscan-maewix"
-    assert summary["artists"][0]["samples"] == [
-        "Libscan-Maewix/patreon/2026-05-01_123_Pack/01_art.jpg",
-    ]
-
-
-def test_queue_makes_a_library_batch_of_what_find_lists(db_sync, layout, monkeypatch):
-    """The import queued is exactly the previewed list."""
-    root, files = layout
-    queued = []
-    monkeypatch.setattr(import_file, "enqueue_import", lambda tid, tt: queued.append((tid, tt)))
-
-    summary = library_scan.queue_unimported(db_sync, root)
-
-    tasks = db_sync.execute(
-        select(ImportTask).where(ImportTask.batch_id == summary["batch_id"])
-    ).scalars().all()
-    assert [t.source_path for t in tasks] == [str(files["missing"])]
-    assert tasks[0].batch.scan_mode == "library"
-    assert queued == [(tasks[0].id, "media")]
-
-
-def test_a_library_import_attaches_in_place_and_removes_a_copy(db_sync, layout, monkeypatch):
-    """The file stays where it is and becomes the record's file. A byte-identical
-    copy found later is removed from disk, since the library already holds it."""
-    root, files = layout
-    monkeypatch.setattr(import_file, "IMAGES_ROOT", root)
-    importer = Importer(
-        session=db_sync, images_root=root, import_root=root,
-        thumbnailer=Thumbnailer(images_root=root),
-        settings=ImportSettings.load_sync(db_sync),
-    )
-
-    result = import_file._attach_where_it_lies(db_sync, importer, files["missing"])
-
-    assert result.status == "imported"
-    record = db_sync.get(ImageRecord, result.image_id)
-    assert record.path == str(files["missing"])
-    assert files["missing"].exists()
-    assert db_sync.get(Artist, record.artist_id).slug == "libscan-maewix"
-
-    copy = _write(files["missing"].parent / "09_copy.jpg", files["missing"].read_bytes())
-    again = import_file._attach_where_it_lies(db_sync, importer, copy)
-    assert again.status == "skipped"
-    assert not copy.exists()
-    assert files["missing"].exists()

tests/test_maintenance.py

@@ -26,50 +26,26 @@ def _make_batch(session) -> int:
     return batch.id
 
 
-def test_cleanup_orphaned_temp_files_removes_stale_only(db_sync, tmp_path, monkeypatch):
+def test_cleanup_orphaned_temp_files_removes_stale_only(tmp_path, monkeypatch):
     import os
 
-    from backend.app.models import ImageRecord, PostAttachment
     from backend.app.tasks import maintenance as m
 
     monkeypatch.setattr(m, "IMAGES_ROOT", tmp_path)
+    stale = tmp_path / "artist" / "img.jpg.part"      # killed download → orphan
+    stale.parent.mkdir(parents=True)
+    stale.write_bytes(b"x")
     old = datetime.now(UTC).timestamp() - 8 * 3600     # older than the 6h guard
-
-    def staged(rel: str, *, aged: bool = True):
-        p = tmp_path / rel
-        p.parent.mkdir(parents=True, exist_ok=True)
-        p.write_bytes(b"x")
-        if aged:
-            os.utime(p, (old, old))
-        return p
-
-    stale = staged("artist/img.jpg.part")                 # killed download → orphan
-    mega = staged("artist/.megatmp.AbCd123")              # abandoned megatools stage
-    fresh = staged("in_progress.jpg.partial", aged=False)  # active download → keep
-    keep = tmp_path / "real.jpg"                          # a real image → keep
+    os.utime(stale, (old, old))
+    fresh = tmp_path / "in_progress.jpg.partial"       # active download → keep
+    fresh.write_bytes(b"x")
+    keep = tmp_path / "real.jpg"                        # a real image → keep
     keep.write_bytes(b"x")
-    # Staging-named files a library row points at are library content.
-    attached = staged("attachments/ab/abc.part")
-    imaged = staged("artist/video.mp4.part")
-    db_sync.add(PostAttachment(
-        post_id=None, artist_id=None, sha256="tempsweep-att".ljust(64, "x"),
-        path=str(attached), original_filename="film.mp4.part", ext=".part",
-        size_bytes=1,
-    ))
-    db_sync.add(ImageRecord(
-        path=str(imaged), sha256="tempsweep-img".ljust(64, "x"), size_bytes=1,
-        mime="video/mp4", width=1, height=1, origin="downloaded",
-        integrity_status="ok",
-    ))
-    db_sync.commit()
 
-    assert m.cleanup_orphaned_temp_files() == 2
+    assert m.cleanup_orphaned_temp_files() == 1
     assert not stale.exists()
-    assert not mega.exists()
     assert fresh.exists()
     assert keep.exists()
-    assert attached.exists()
-    assert imaged.exists()
 
 
 def test_recover_interrupted_only_old(db_sync, monkeypatch):
@@ -122,10 +98,7 @@ def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypat
     Banked 2026-05-25 after operator hit 5490 stuck pending rows.
     """
     from backend.app.tasks import import_file
-    from backend.app.tasks import maintenance as m
     monkeypatch.setattr(import_file.import_media_file, "delay", lambda *_: None)
-    # The broker's import queue is empty, so a stale 'queued' row's message is gone.
-    monkeypatch.setattr(m, "_import_queue_depth", lambda: 0)
 
     batch_id = _make_batch(db_sync)
     now = datetime.now(UTC)
@@ -173,42 +146,6 @@ def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypat
     assert "orphan" in (stale_pending.error or "")
 
 
-@pytest.mark.parametrize("depth", [7, None])
-def test_recover_interrupted_keeps_queued_rows_while_the_import_queue_holds_work(
-    db_sync, monkeypatch, depth,
-):
-    """A queued row in a batch bigger than half an hour of work is waiting its
-    turn, not orphaned: while the import queue holds messages (or can't be
-    read), only never-enqueued 'pending' rows are swept."""
-    from sqlalchemy import update as _upd
-
-    from backend.app.tasks import maintenance as m
-
-    monkeypatch.setattr(m, "_import_queue_depth", lambda: depth)
-    batch_id = _make_batch(db_sync)
-    waiting = ImportTask(
-        batch_id=batch_id, source_path="/import/waiting.jpg", task_type="media",
-        status="queued",
-    )
-    never_sent = ImportTask(
-        batch_id=batch_id, source_path="/import/never.jpg", task_type="media",
-        status="pending",
-    )
-    db_sync.add_all([waiting, never_sent])
-    db_sync.flush()
-    db_sync.execute(
-        _upd(ImportTask).where(ImportTask.id.in_([waiting.id, never_sent.id]))
-        .values(created_at=datetime.now(UTC) - timedelta(hours=2))
-    )
-    db_sync.commit()
-
-    assert m.recover_interrupted_tasks.apply().get() == 1
-    db_sync.refresh(waiting)
-    db_sync.refresh(never_sent)
-    assert waiting.status == "queued"
-    assert never_sent.status == "failed"
-
-
 def test_recover_interrupted_handles_both_stuck_and_orphans(db_sync, monkeypatch):
     """One sweep tick handles both 'processing' crashes AND
     'pending'/'queued' orphans in a single pass."""
@@ -948,14 +885,3 @@ def test_reclaim_attachments_stuck_threshold_exceeds_hard_time_limit():
         "backend.app.tasks.admin.reclaim_orphaned_attachments_task"
     ]
     assert override >= hard_minutes
-
-
-def test_import_unimported_stuck_threshold_exceeds_hard_time_limit():
-    """The same invariant for the walk that imports files on disk."""
-    from backend.app.tasks.admin import import_unimported_files_task
-    from backend.app.tasks.maintenance import TASK_STUCK_THRESHOLD_MINUTES
-
-    override = TASK_STUCK_THRESHOLD_MINUTES[
-        "backend.app.tasks.admin.import_unimported_files_task"
-    ]
-    assert override >= import_unimported_files_task.time_limit / 60

tests/test_tasks_backup.py

@@ -33,7 +33,7 @@ def fake_subprocess_and_images_root(monkeypatch, tmp_path):
         stderr = b""
 
     def _sentinel(cmd):
-        if "pg_dump" in cmd:
+        if cmd[0] == "pg_dump":
             i = cmd.index("-f")
             Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
         elif cmd[0] == "tar" and "-cf" in cmd:
@@ -128,63 +128,6 @@ async def test_backup_db_task_persists_tag(db_sync):
     assert tag == "pre-cutover"
 
 
-@pytest.fixture
-def archives(monkeypatch):
-    """An off-site destination is set up; the dump itself is faked."""
-    from backend.app.services.backup_s3 import S3Target
-    target = S3Target("http://archives.example:8081", "curator", "AK", "SK")
-    monkeypatch.setattr(
-        "backend.app.services.backup_destination.saved_target", lambda settings: target,
-    )
-    sent = []
-
-    def _offsite(*, db_url, target, limit_mbps):
-        sent.append({"target": target, "limit_mbps": limit_mbps})
-        return {
-            "kind": "db", "ts": "20261107T030000Z",
-            "key": "database/fabledcurator-20261107T030000Z.dump",
-            "bucket": target.bucket, "endpoint": target.endpoint,
-            "parts": 3, "size_bytes": 1234,
-        }
-    monkeypatch.setattr("backend.app.services.backup_service.backup_db_offsite", _offsite)
-    return sent
-
-
-@pytest.mark.asyncio
-async def test_with_a_destination_the_dump_goes_off_site_and_nothing_is_written_here(
-    db_sync, archives, tmp_path,
-):
-    from backend.app.tasks.backup import backup_db_task
-    run_id = backup_db_task.delay(triggered_by="nightly").get()["backup_run_id"]
-    row = db_sync.get(BackupRun, run_id)
-    assert row.status == "ok"
-    assert row.sql_path is None
-    assert row.size_bytes == 1234
-    assert row.manifest["offsite"] == {
-        "endpoint": "http://archives.example:8081", "bucket": "curator",
-        "key": "database/fabledcurator-20261107T030000Z.dump", "parts": 3,
-    }
-    assert archives[0]["limit_mbps"] == 200        # the setting's default
-    assert not (tmp_path / "_backups").exists()
-
-
-@pytest.mark.asyncio
-async def test_an_off_site_dump_that_fails_is_recorded_with_why(db_sync, monkeypatch, archives):
-    from backend.app.services.backup_service import DumpFailed
-    from backend.app.tasks.backup import backup_db_task
-
-    def _fails(**kwargs):
-        raise DumpFailed("pg_dump: error: connection lost")
-    monkeypatch.setattr("backend.app.services.backup_service.backup_db_offsite", _fails)
-    with pytest.raises(DumpFailed):
-        backup_db_task.delay().get()
-    row = db_sync.execute(
-        select(BackupRun).where(BackupRun.kind == "db").order_by(BackupRun.id.desc()).limit(1)
-    ).scalar_one()
-    assert row.status == "error"
-    assert row.error == "DumpFailed: pg_dump: error: connection lost"
-
-
 # --- backup_images_task ---------------------------------------------