Merge pull request 'v26.05.26.2 — hotfix: alembic 0022 Post-collision pre-merge + ci.yml cache continue-on-error' (#24) from dev into main

pg_dump + zstd in runtime image, lift Quart body cap to 1 GiB. See PR #2.

.forgejo/workflows/build.yml

@@ -2,17 +2,7 @@ name: Build images
 
 on:
   push:
-    # `:dev` builds dropped 2026-05-26 — operator tests from `:latest` after
-    # merge-to-main, not from the dev branch image. Saves one full docker
-    # build per dev push.
-    branches: [main]
-    # Tag-push triggers an immutable per-version image build (e.g.
-    # `:v26.05.26.5`) — gives a real rollback story alongside the floating
-    # `:main` / `:latest`. Layer reuse keeps the registry-storage cost
-    # negligible per tag. Doesn't overlap with the push-to-main build (that
-    # one publishes `:main` + `:latest`; the tag-push build publishes only
-    # `:<tag>`).
-    tags: ['v*']
+    branches: [dev, main]
 
 # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
 #   - write:package, read:package  (for docker push to git.fabledsword.com)
@@ -168,56 +158,25 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Download signed XPI from Forgejo release asset (main + tags)
-        # Fires on main-push AND on tag-push. Tag-push builds re-package the
-        # same source code as the preceding main-push build but with an
-        # immutable version tag — they need the XPI too, otherwise the
-        # versioned image ships without the signed extension.
-        #
-        # Tag-push vs main-push race (operator-flagged 2026-05-27 after
-        # v26.05.27.0 hit it): a release cut fires BOTH workflows almost
-        # simultaneously. Main-push runs sign-extension (1-5min AMO round
-        # trip) before publishing the ext-<version> release; tag-push
-        # skips sign-extension (gated to main) and races straight to
-        # this download step. Tag-push lost every time. Fix: poll the
-        # ext-<version> release endpoint with a sleep+retry loop (30s
-        # for up to 10min total) before giving up. Main-push's signing
-        # eventually wins and tag-push picks the release up on a later
-        # iteration.
-        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+      - name: Download signed XPI from Forgejo release asset (main only)
+        if: github.ref == 'refs/heads/main'
         env:
           TOKEN: ${{ secrets.RELEASE_TOKEN }}
         run: |
           set -eux
           VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
-          # Poll for the ext-<version> release. main-push's sign-extension
-          # step (AMO round-trip, 1-5min) needs to finish + upload before
-          # tag-push can fetch. 30s * 20 = up to 10min wait, then hard-fail.
-          for attempt in $(seq 1 20); do
-            STATUS=$(curl -s -o release.json -w "%{http_code}" \
-              -H "Authorization: token $TOKEN" \
-              "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
-            if [ "$STATUS" = "200" ]; then
-              echo "Found ext-$VERSION release on attempt $attempt"
-              break
-            fi
-            if [ "$attempt" = "20" ]; then
-              echo "ERROR: ext-$VERSION release not available after 10min of polling"
-              echo "Last HTTP status: $STATUS"
-              exit 1
-            fi
-            echo "Attempt $attempt: ext-$VERSION not yet published (HTTP $STATUS); sleeping 30s"
-            sleep 30
-          done
-          # Extract the .xpi asset's browser_download_url (Forgejo's
-          # /releases/assets/<id> endpoint returns ASSET METADATA, not
-          # the binary blob — operator-flagged 2026-05-26: my prior
-          # code curl'd the metadata endpoint without -f and wrote the
-          # resulting 404-page-not-found text into fabledcurator-*.xpi,
-          # which Firefox then rejected as "corrupt").
-          # browser_download_url is the canonical binary endpoint and
-          # is also publicly accessible (no token needed) but we pass
-          # the token anyway for symmetry with private-repo support.
+          # Look up the ext-<version> release; extract the .xpi asset's
+          # browser_download_url (Forgejo's /releases/assets/<id> endpoint
+          # returns ASSET METADATA, not the binary blob — operator-flagged
+          # 2026-05-26: my prior code curl'd the metadata endpoint without
+          # -f and wrote the resulting 404-page-not-found text into
+          # fabledcurator-*.xpi, which Firefox then rejected as "corrupt").
+          # browser_download_url is the canonical binary endpoint and is
+          # also publicly accessible (no token needed) but we pass the
+          # token anyway for symmetry with private-repo support.
+          curl -sf -H "Authorization: token $TOKEN" \
+            "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" \
+            -o release.json
           DOWNLOAD_URL=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['browser_download_url'])")
           test -n "$DOWNLOAD_URL"
           echo "Downloading XPI from: $DOWNLOAD_URL"
@@ -241,33 +200,8 @@ jobs:
       - name: Determine tag
         id: tag
         run: |
-          # Three trigger shapes:
-          #   refs/tags/v…   → tag-push: opt-in milestone label (vYY.MM.DD,
-          #                    no `.N` per family release-posture rule).
-          #                    Publish ONLY the immutable version tag;
-          #                    don't touch :latest (the main-push build
-          #                    for the merge commit already did that).
-          #   refs/heads/main → push to main: publish :main + :latest
-          #                    (floating) AND :c-<short_sha> (immutable
-          #                    per-commit rollback substrate, per family
-          #                    release-posture rule "Tags are milestones,
-          #                    not gates — commit-SHA images are the
-          #                    rollback unit"). Rollback to any commit
-          #                    becomes `docker pull …:c-<sha>` without a
-          #                    release ceremony.
-          #   anything else  → safety net; shouldn't fire given the `on:`
-          #                    config above. Tag :dev to surface the
-          #                    unexpected run in the registry.
-          # POSIX-safe substring (the runner shell is dash/BusyBox sh, not
-          # bash — `${var:0:7}` errors with "Bad substitution"; cut works
-          # everywhere). Operator-flagged 2026-06-01 after first :c-<sha>
-          # main-push build failed at this step.
-          SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
-          if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
-            TAG_NAME="${GITHUB_REF#refs/tags/}"
-            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
-          elif [ "${GITHUB_REF##*/}" = "main" ]; then
-            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          if [ "${GITHUB_REF##*/}" = "main" ]; then
+            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
           else
             echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
           fi
@@ -297,20 +231,8 @@ jobs:
       - name: Determine tag
         id: tag
         run: |
-          # Mirrors build-web's three-shape logic (tag-push / main-push /
-          # safety-net dev) including the per-commit :c-<short_sha> tag
-          # on main-push per the family release-posture rule. The -ml
-          # image follows the same release cadence as the web image.
-          # POSIX-safe substring (the runner shell is dash/BusyBox sh, not
-          # bash — `${var:0:7}` errors with "Bad substitution"; cut works
-          # everywhere). Operator-flagged 2026-06-01 after first :c-<sha>
-          # main-push build failed at this step.
-          SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
-          if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
-            TAG_NAME="${GITHUB_REF#refs/tags/}"
-            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
-          elif [ "${GITHUB_REF##*/}" = "main" ]; then
-            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          if [ "${GITHUB_REF##*/}" = "main" ]; then
+            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
           else
             echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
           fi
@@ -329,41 +251,3 @@ jobs:
           file: Dockerfile.ml
           push: true
           tags: ${{ steps.tag.outputs.tags }}
-
-  # The desktop GPU agent (#114) — published so the operator pulls + runs it on
-  # the GPU machine instead of building locally. Independent of web/ml (its own
-  # CUDA + onnxruntime-gpu image, context = agent/). Same tag cadence.
-  build-agent:
-    runs-on: python-ci
-    container:
-      image: git.fabledsword.com/bvandeusen/ci-python:3.14
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Determine tag
-        id: tag
-        run: |
-          SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
-          if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
-            TAG_NAME="${GITHUB_REF#refs/tags/}"
-            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:${TAG_NAME}" >> "$GITHUB_OUTPUT"
-          elif [ "${GITHUB_REF##*/}" = "main" ]; then
-            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:main,git.fabledsword.com/bvandeusen/fabledcurator-agent:latest,git.fabledsword.com/bvandeusen/fabledcurator-agent:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
-          else
-            echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:dev" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Login to Forgejo registry
-        uses: docker/login-action@v3
-        with:
-          registry: git.fabledsword.com
-          username: ${{ github.actor }}
-          password: ${{ secrets.RELEASE_TOKEN }}
-
-      - name: Build and push agent image
-        uses: docker/build-push-action@v5
-        with:
-          context: agent
-          file: agent/Dockerfile
-          push: true
-          tags: ${{ steps.tag.outputs.tags }}

.forgejo/workflows/ci.yml

@@ -1,41 +1,17 @@
 name: CI
 
 # CI lanes per FabledRulebook/forgejo.md "CI philosophy":
-#   - lint: ruff only, no dep install — fast-fail for the common lint bounce.
-#   - backend-lint-and-test: `pytest -m "not integration"`, no service containers.
+#   - backend-lint-and-test: ruff + `pytest -m "not integration"`, no service containers.
 #   - frontend-build: vitest unit + vite build.
 #   - integration: pgvector + redis service containers; alembic + `pytest -m integration`.
 
 on:
   push:
     branches: [dev, main]
-  # pull_request trigger intentionally absent — with branches: [dev, main]
-  # above, every PR commit already fires CI via the push event on dev. Adding
-  # pull_request would duplicate runs on dev→main PRs. FC has no fork PRs
-  # (single-operator Forgejo repo) so push coverage is complete.
+  pull_request:
+    branches: [main]
 
 jobs:
-  # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
-  # this runs with NO dependency install and surfaces the most common bounce
-  # class (lint: I001 / UP037 / ASYNC109 / W293 …) in seconds — instead of
-  # after the backend job's ~30-60s wheel install. ruff is static analysis,
-  # so no DB/secret env is needed.
-  lint:
-    runs-on: python-ci
-    container:
-      image: git.fabledsword.com/bvandeusen/ci-python:3.14
-    steps:
-      - uses: actions/checkout@v4
-      - name: Ruff lint
-        # agent/ included so the GPU-agent is linted before its image is built
-        # (build.yml only `docker build`s it — this is where it gets checked).
-        run: ruff check backend/ tests/ alembic/ agent/
-      - name: Agent syntax check
-        # The agent's runtime deps (torch/transformers/ultralytics) aren't in the
-        # CI image, so we can't import it — but compileall parses every module,
-        # catching syntax errors before the image build.
-        run: python -m compileall -q agent/fc_agent
-
   backend-lint-and-test:
     runs-on: python-ci
     container:
@@ -48,17 +24,22 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      # Cache step removed 2026-05-26: act_runner's cache backend has been
-      # broken on this homelab runner since 2026-05-15 (first as request-
-      # timeout warnings, then as hard "Cannot find module .../dist/restore/
-      # index.js" failures that tank the whole job). The cache step targeted
-      # ~/.cache/pip but the install below uses `uv pip install` primarily,
-      # whose own cache lives at ~/.cache/uv — so the cache step's real
-      # benefit was marginal even when working. Cost of removal: ~30s of
-      # wheel downloads per job. Future re-enable: mount ~/.cache/uv as a
-      # docker volume at the runner level (skips actions/cache entirely),
-      # or fix the runner-side cache backend (clear /var/run/act/actions/*,
-      # pin act_runner version, etc.).
+      - name: Cache pip wheels
+        # continue-on-error: act_runner's cache backend has been broken on
+        # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
+        # but the action's JS bundle now fails to load at all
+        # ("Cannot find module .../dist/restore/index.js"), hard-failing
+        # the whole job. The install step that follows handles cold caches
+        # natively (uv pip / pip install both work without it), so net
+        # cost of disabling the cache here is ~30s of wheel downloads per
+        # job. Re-enable once the runner-side cache backend is fixed.
+        continue-on-error: true
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
 
       - name: Install Python deps
         # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
@@ -73,8 +54,9 @@ jobs:
             pip install -r requirements.txt pytest pytest-asyncio
           fi
 
-      # Ruff moved to the dedicated fast `lint` job above (fails in seconds,
-      # no dep install). This job is now unit tests only.
+      - name: Ruff lint
+        run: ruff check backend/ tests/ alembic/
+
       - name: Pytest (unit only — integration runs in the integration job)
         run: pytest tests/ -v -m "not integration"
 
@@ -99,25 +81,28 @@ jobs:
       - run: npm run test:unit
       - run: npm run build
 
-  # Single integration job — collapsed from a 3-way shard split on 2026-06-04.
-  # The shards existed to parallelize ~8.5min of integration tests; once the
-  # throwaway Postgres runs with fsync OFF (the durability step below) the whole
-  # suite runs in ~45s, so the split only triplicated the ~2min fixed overhead
-  # (container + `uv pip install` + `alembic upgrade head`) and burned 3 of 6
-  # runner slots for no wall-clock gain. One job now: spin up once, install
-  # once, migrate once, run every integration test.
+  # Integration suite split into THREE parallel shards (2026-05-25, runner
+  # capacity bumped 2→6). Each shard gets its own Postgres + Redis service
+  # set and runs alembic + a disjoint subset of integration tests. Shards
+  # share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
+  # stays single-threaded per shard but multiple shards run in parallel
+  # wall-clock. Approximate split — rebalance once --durations=15 output
+  # reveals which shard is the long pole.
   #
-  # The docker-ps filter scopes to THIS job's own Postgres/Redis service
-  # containers by job name. act_runner strips underscores from job names when
-  # labelling containers (`int_api` matched nothing on 2026-05-25), so the name
-  # stays separator-free (`integration`). The step prints `docker ps -a` first
-  # so a future naming-convention shift surfaces in the log without a
-  # guess-and-push cycle.
+  # Each shard's docker-ps filter uses its own unique job name to scope
+  # service-container resolution. act_runner appears to strip underscores
+  # from job names when building container labels — `int_api` yielded
+  # zero matches on 2026-05-25 — so shards use no-separator names
+  # (`intapi`, `intimp`, `intcore`) instead. Each step prints
+  # `docker ps -a` first so a future naming-convention shift surfaces in
+  # the log without another guess-and-push cycle.
   #
-  # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT done —
-  # per ci-requirements.md, FC is the only Python consumer of that image and the
-  # CI-Runner "add deps to image when used by >1 project" rule keeps it per-job.
-  integration:
+  # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
+  # done — per ci-requirements.md, FC is the only Python consumer of that
+  # image and the CI-Runner project's "add deps to image when used by >1
+  # project" rule keeps the install per-job.
+
+  intapi:
     runs-on: python-ci
     container:
       image: git.fabledsword.com/bvandeusen/ci-python:3.14
@@ -148,14 +133,30 @@ jobs:
           --health-retries 10
     steps:
       - uses: actions/checkout@v4
-      - name: Integration suite (resolve service IPs, migrate, test)
+      - name: Cache pip wheels
+        # continue-on-error: act_runner's cache backend has been broken on
+        # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
+        # but the action's JS bundle now fails to load at all
+        # ("Cannot find module .../dist/restore/index.js"), hard-failing
+        # the whole job. The install step that follows handles cold caches
+        # natively (uv pip / pip install both work without it), so net
+        # cost of disabling the cache here is ~30s of wheel downloads per
+        # job. Re-enable once the runner-side cache backend is fixed.
+        continue-on-error: true
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
+      - name: API integration shard (resolve service IPs, migrate, test)
         run: |
           set -eux
           echo "=== container landscape (diagnostic for filter scoping) ==="
           docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
           echo "=== end landscape ==="
-          PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
+          PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+          RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1)
           test -n "$PG" && test -n "$RD"
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
@@ -172,14 +173,162 @@ jobs:
           else
             pip install -r requirements.txt pytest pytest-asyncio
           fi
-          # Relax durability on the throwaway CI Postgres so the per-test
-          # TRUNCATE's commit-fsync — the integration teardown's dominant cost
-          # (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is
-          # skipped. fsync/full_page_writes are sighup GUCs and synchronous_commit
-          # is user-context, so ALTER SYSTEM + pg_reload_conf() applies them with
-          # NO restart. Ephemeral DB ⇒ fsync-off is safe. Non-fatal so a perms
-          # surprise can't red the job; fabledcurator is the postgres image's
-          # bootstrap superuser.
-          python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)'
           alembic upgrade head
-          pytest tests/ -v -m integration --durations=15
+          pytest tests/test_api_*.py -v -m integration --durations=15
+
+  intimp:
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    env:
+      DB_USER: fabledcurator
+      DB_PASSWORD: ci_integration
+      DB_PORT: "5432"
+      DB_NAME: fabledcurator_test
+      SECRET_KEY: ci_integration_placeholder
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: fabledcurator
+          POSTGRES_PASSWORD: ci_integration
+          POSTGRES_DB: fabledcurator_test
+        options: >-
+          --health-cmd "pg_isready -U fabledcurator"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache pip wheels
+        # continue-on-error: act_runner's cache backend has been broken on
+        # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
+        # but the action's JS bundle now fails to load at all
+        # ("Cannot find module .../dist/restore/index.js"), hard-failing
+        # the whole job. The install step that follows handles cold caches
+        # natively (uv pip / pip install both work without it), so net
+        # cost of disabling the cache here is ~30s of wheel downloads per
+        # job. Re-enable once the runner-side cache backend is fixed.
+        continue-on-error: true
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
+      - name: Importer integration shard (resolve service IPs, migrate, test)
+        run: |
+          set -eux
+          echo "=== container landscape (diagnostic for filter scoping) ==="
+          docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+          echo "=== end landscape ==="
+          PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+          RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1)
+          test -n "$PG" && test -n "$RD"
+          PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
+          RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
+          test -n "$PG_IP" && test -n "$RD_IP"
+          export DB_HOST="$PG_IP"
+          export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
+          export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
+          for i in $(seq 1 60); do
+            (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
+            sleep 2
+          done
+          if command -v uv >/dev/null 2>&1; then
+            uv pip install --system -r requirements.txt pytest pytest-asyncio
+          else
+            pip install -r requirements.txt pytest pytest-asyncio
+          fi
+          alembic upgrade head
+          pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
+
+  intcore:
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    env:
+      DB_USER: fabledcurator
+      DB_PASSWORD: ci_integration
+      DB_PORT: "5432"
+      DB_NAME: fabledcurator_test
+      SECRET_KEY: ci_integration_placeholder
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: fabledcurator
+          POSTGRES_PASSWORD: ci_integration
+          POSTGRES_DB: fabledcurator_test
+        options: >-
+          --health-cmd "pg_isready -U fabledcurator"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache pip wheels
+        # continue-on-error: act_runner's cache backend has been broken on
+        # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
+        # but the action's JS bundle now fails to load at all
+        # ("Cannot find module .../dist/restore/index.js"), hard-failing
+        # the whole job. The install step that follows handles cold caches
+        # natively (uv pip / pip install both work without it), so net
+        # cost of disabling the cache here is ~30s of wheel downloads per
+        # job. Re-enable once the runner-side cache backend is fixed.
+        continue-on-error: true
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
+      - name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
+        run: |
+          set -eux
+          echo "=== container landscape (diagnostic for filter scoping) ==="
+          docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+          echo "=== end landscape ==="
+          PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+          RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1)
+          test -n "$PG" && test -n "$RD"
+          PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
+          RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
+          test -n "$PG_IP" && test -n "$RD_IP"
+          export DB_HOST="$PG_IP"
+          export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
+          export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
+          for i in $(seq 1 60); do
+            (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
+            sleep 2
+          done
+          if command -v uv >/dev/null 2>&1; then
+            uv pip install --system -r requirements.txt pytest pytest-asyncio
+          else
+            pip install -r requirements.txt pytest pytest-asyncio
+          fi
+          alembic upgrade head
+          pytest tests/ -v -m integration --durations=15 \
+            --ignore-glob='tests/test_api_*.py' \
+            --ignore-glob='tests/test_importer*.py' \
+            --ignore-glob='tests/test_import_*.py' \
+            --ignore-glob='tests/test_migration_*.py' \
+            --ignore-glob='tests/test_phash_*.py' \
+            --ignore-glob='tests/test_sidecar_*.py' \
+            --ignore-glob='tests/test_scan_*.py' \
+            --ignore-glob='tests/test_archive_extractor.py' \
+            --ignore-glob='tests/test_backfill_phash.py'

.gitignore

@@ -61,12 +61,8 @@ Thumbs.db
 
 # Claude Code per-user local overrides (shared .claude/settings.json is OK to commit)
 .claude/settings.local.json
-# Transient scheduler lock/state (committed by accident in 3f30327)
-.claude/scheduled_tasks.lock
-.claude/scheduled_tasks*.json
 
 # Alembic / DB scratch
 alembic/versions/__pycache__/
 *.sqlite
 *.sqlite-journal
-.superpowers/

Dockerfile

@@ -18,16 +18,13 @@ ENV PYTHONUNBUFFERED=1 \
 
 # System deps: ffmpeg (transcode + thumbnails, FC-2), unar (archives, FC-2),
 # libpq for psycopg, postgresql-client + zstd for FC-5 backup/restore
-# (pg_dump + tar --zstd), image libs, megatools (mega.nz public-link downloads
-# for off-platform file-host links, #830 — `megatools dl`; Debian-native, no
-# external MEGA apt repo needed).
+# (pg_dump + tar --zstd), image libs.
 RUN apt-get update && apt-get install -y --no-install-recommends \
         ffmpeg \
         unar \
         libpq5 \
         postgresql-client \
         zstd \
-        megatools \
         libjpeg62-turbo \
         libwebp7 \
         libpng16-16 \

agent/Dockerfile

@@ -1,33 +0,0 @@
-# FabledCurator GPU agent — runs on the desktop with the GPU.
-# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs
-# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open
-# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
-# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are
-# built against (CUDA 13 has only nascent ONNX Runtime support).
-FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04
-
-# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
-# managed (PEP 668), so a global `pip install` errors without this. It's a
-# single-purpose container — we own the whole environment, so installing into
-# the system site-packages is fine (and simplest — no venv on PATH to manage).
-ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1 PIP_BREAK_SYSTEM_PACKAGES=1
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends python3 python3-pip ffmpeg \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /app
-# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN
-# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first
-# + separately so the GPU build of torch is deterministic and layer-cached.
-RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
-COPY requirements.txt .
-RUN pip3 install --no-cache-dir -r requirements.txt
-COPY fc_agent ./fc_agent
-
-# imgutils ONNX models + the transformers SigLIP weights both cache here; mount
-# a volume to persist them across restarts (the SigLIP download is ~3.5 GB once).
-ENV HF_HOME=/models
-EXPOSE 8770
-
-# The control UI; the worker is started from it (or POST /start).
-CMD ["uvicorn", "fc_agent.app:app", "--host", "0.0.0.0", "--port", "8770"]

agent/README.md

@@ -1,71 +0,0 @@
-# FabledCurator GPU agent
-
-A desktop-GPU worker that embeds characters (CCIP) + figure crops for
-FabledCurator. It talks to FC **only over HTTP** — it leases jobs, fetches image
-pixels, runs the models on your GPU, and posts results back. Your FC database and
-Redis stay private; the agent never touches them.
-
-You run it when you want a burst and stop it to reclaim the card.
-
-## 0. Host prerequisite — NVIDIA Container Toolkit
-Docker needs the toolkit to hand the GPU to a container (else: *"could not select
-device driver nvidia with capabilities [[gpu]]"*). On Arch/CachyOS:
-```sh
-sudo pacman -S nvidia-container-toolkit
-sudo nvidia-ctk runtime configure --runtime=docker
-sudo systemctl restart docker
-# verify:
-docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi
-```
-
-## 1. Get a token
-In FC: **Settings → Tagging → GPU agent → Generate token** (or Rotate). Copy it.
-
-## 2. Pull (CI publishes it alongside the web/ml images)
-```sh
-docker pull git.fabledsword.com/bvandeusen/fabledcurator-agent:latest
-```
-> Local build for development instead: `docker build -t fc-gpu-agent agent/`
-
-## 3. Run (on the machine with the GPU)
-```sh
-docker run --rm --gpus all -p 8770:8770 \
-  -e FC_URL=http://curator.traefik.internal \
-  -e FC_TOKEN=<paste-the-token> \
-  -v fc-agent-models:/models \
-  git.fabledsword.com/bvandeusen/fabledcurator-agent:latest
-```
-Then open <http://localhost:8770> — the control page. Click **Start** to begin
-draining the queue; **Pause**/**Stop** to yield the GPU. The `-v fc-agent-models`
-volume caches the downloaded ONNX models so restarts are fast.
-
-Kick off a backfill from FC (**GPU agent card → Queue character embedding**), then
-watch the queue counts on the control page (or FC's card) drain.
-
-## Config (env)
-| var | default | meaning |
-|---|---|---|
-| `FC_URL` | `http://localhost:8000` | FC base URL |
-| `FC_TOKEN` | — | the bearer token (required) |
-| `AGENT_ID` | `desktop-agent` | identifies this agent's leases |
-| `BATCH_SIZE` | `4` | jobs leased per round (still processed one at a time) |
-| `CCIP_MODEL` | imgutils default | CCIP model name |
-| `DETECTOR_LEVEL` | `m` | person-detector size: `n` < `s` < `m` < `x` |
-| `POLL_IDLE_SECONDS` | `10` | wait between empty leases |
-
-## ⚠️ Verify on first run
-This part can't be CI-tested (no GPU/models in CI), so confirm against your
-installed `dghs-imgutils` (`pip show dghs-imgutils`) — see `fc_agent/models.py`:
-- `imgutils.detect.detect_person(image, level=...)` returns
-  `[((x0,y0,x1,y1), label, score), ...]`.
-- `imgutils.metrics.ccip_extract_feature(image, model=...)` returns a vector
-  (768-d for caformer). If you want the F1-0.94 variant, set
-  `CCIP_MODEL=ccip-caformer_b36-24` (verify the exact string in imgutils).
-
-If FC's matcher under/over-fires, tune the cosine threshold in
-`backend/app/services/ml/ccip.py` (`DEFAULT_SIM_THRESHOLD`) and use
-`GET /api/ccip/overview` + `/api/ccip/images/<id>` to spot-check.
-
-## CPU fallback
-Swap `onnxruntime-gpu` → `onnxruntime` in `requirements.txt` and drop `--gpus all`
-to grind it slowly on the server instead. Same agent, no card.

agent/docker-compose.yml

@@ -1,73 +0,0 @@
-# FabledCurator GPU agent — desktop run via docker compose.
-#
-# Usage:
-#   1. Generate a token: FC → Settings → Tagging → GPU agent → Generate token.
-#   2. Create a .env next to this file:
-#        FC_URL=http://curator.traefik.internal
-#        FC_TOKEN=<paste-the-token>
-#        # optional: CCIP_MODEL=ccip-caformer_b36-24   (the F1-0.94 variant)
-#   3. docker compose up -d     (pulls the published image)
-#   4. Open http://localhost:8770 → Start. Pause/Stop hands the GPU back.
-#      docker compose down       to stop the container entirely.
-#
-# Surviving a curator redeploy (you're away, can't touch the agent):
-#   - A running agent rides out curator being unreachable on its own — it retries
-#     leasing with capped backoff and resumes when the server is back. In-flight
-#     work is handed back (not failed), so a redeploy never poisons good jobs.
-#   - AUTO_START=1 (below) also resumes the worker if the AGENT container itself
-#     restarts (host reboot / crash via `restart: unless-stopped`) — no click.
-#
-# Needs the NVIDIA Container Toolkit installed on the host for --gpus.
-
-services:
-  fc-gpu-agent:
-    image: git.fabledsword.com/bvandeusen/fabledcurator-agent:latest
-    pull_policy: always
-    ports:
-      - "8770:8770"
-    environment:
-      FC_URL: ${FC_URL:-http://curator.traefik.internal}
-      FC_TOKEN: ${FC_TOKEN:?set FC_TOKEN in .env (FC → GPU agent → Generate token)}
-      CCIP_MODEL: ${CCIP_MODEL:-}
-      DETECTOR_LEVEL: ${DETECTOR_LEVEL:-m}
-      BATCH_SIZE: ${BATCH_SIZE:-4}
-      # Resume the worker automatically on container start (survive a reboot /
-      # crash-restart while you're away). Set to 0 to require a manual Start.
-      AUTO_START: ${AUTO_START:-1}
-      # Autoscale the worker count (throughput hill-climb that finds the sweet
-      # spot + backs off under VRAM pressure). On by default; toggle live in the
-      # control UI. Set to 0 to start in manual mode.
-      AUTO_SCALE: ${AUTO_SCALE:-1}
-      # Aggregate download cap in MB/s (stills + video streams combined), so the
-      # agent can't saturate the desktop's network and wreck browsing — WiFi
-      # especially. 0 = unlimited; tunable live in the control UI.
-      BANDWIDTH_LIMIT_MB_S: ${BANDWIDTH_LIMIT_MB_S:-8}
-      # Crop embedder (SigLIP concept bag): float16 keeps VRAM low on a shared
-      # desktop GPU; the model itself is announced by the server.
-      SIGLIP_DTYPE: ${SIGLIP_DTYPE:-float16}
-      # Crop PROPOSERS (extra YOLO detectors → more/better concept crops). Each
-      # downloads its weights once (cached on the models volume) and self-disables
-      # if the download/load fails. Blank any one to turn it off.
-      #   PERSON_WEIGHTS: general COCO person detector (Western/realistic figures),
-      #     merged with the anime detector. yolo11n.pt (~6 MB, auto-downloaded).
-      #   ANATOMY_WEIGHTS: booru_yolo anime/furry/NSFW components (~40 MB). NB the
-      #     repo states no license — fine for private use. yolov8n_as01.pt is the
-      #     6 MB nano if you want lighter than yolov11m_aa22.pt.
-      #   PANEL_WEIGHTS: mosesb comic-panel detector (Apache-2.0), "hf_repo::file".
-      PERSON_WEIGHTS: ${PERSON_WEIGHTS:-yolo11n.pt}
-      ANATOMY_WEIGHTS: ${ANATOMY_WEIGHTS:-https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt}
-      PANEL_WEIGHTS: ${PANEL_WEIGHTS:-mosesb/best-comic-panel-detection::best.pt}
-    volumes:
-      # Persist the downloaded ONNX models so restarts are fast.
-      - fc-agent-models:/models
-    restart: unless-stopped
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: all
-              capabilities: [gpu]
-
-volumes:
-  fc-agent-models:

agent/fc_agent/app.py

@@ -1,404 +0,0 @@
-"""FastAPI control surface for the agent (served on localhost).
-
-Start / stop the download→GPU pipeline, tune the downloader count live (the
-workload is download-bound, so downloaders are the dial that trades desktop
-bandwidth for throughput), and watch GPU load + buffer occupancy + progress +
-the server-side queue. Config is env-seeded; the downloader count is adjustable
-here on the fly (GPU consumers autoscale between 1 and 2 on their own).
-"""
-import logging
-
-from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, JSONResponse
-
-from . import logbuf
-from .config import Config
-from .gpu import read_gpu
-from .worker import Worker
-
-log = logging.getLogger("fc_agent.app")
-
-# Bump on every agent change. The page embeds this and /status reports it; the UI
-# warns to reload when they differ — so a stale browser-cached page can't be
-# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
-VERSION = "2026-07-02.6 · sleep mode: an empty queue sheds to one downloader and backs the lease poll off to 15 min"
-
-logbuf.install()
-cfg = Config.from_env()
-worker = Worker(cfg)
-app = FastAPI(title="FabledCurator GPU agent")
-
-
-@app.middleware("http")
-async def _no_store(request, call_next):
-    # The control page is a static string and the status/gpu/logs polls are
-    # live data — never let the browser cache either, or a freshly-pulled agent
-    # image still shows the OLD UI until a hard refresh (operator-flagged
-    # 2026-06-30).
-    resp = await call_next(request)
-    resp.headers["Cache-Control"] = "no-store"
-    return resp
-
-
-@app.on_event("startup")
-def _maybe_autostart() -> None:
-    # With AUTO_START set, a container restart (host reboot, or `restart:
-    # unless-stopped` after a crash) resumes the worker on its own — the slots
-    # then ride out a still-down curator via lease backoff. Lets the agent
-    # survive a redeploy with nobody at the desktop to click Start.
-    if cfg.auto_start and cfg.token:
-        worker.start()
-
-
-@app.get("/", response_class=HTMLResponse)
-def index() -> str:
-    return _PAGE.replace("__BUILD__", VERSION)
-
-
-@app.post("/start")
-def start():
-    log.info("UI: Start button pressed")   # the press; worker logs the transition
-    worker.start()
-    return JSONResponse(worker.status())
-
-
-@app.post("/stop")
-def stop():
-    log.info("UI: Stop button pressed")
-    worker.stop()
-    return JSONResponse(worker.status())
-
-
-@app.post("/concurrency")
-async def concurrency(request: Request):
-    body = await request.json()
-    worker.set_concurrency(int(body.get("value", 1)))
-    return JSONResponse(worker.status())
-
-
-@app.post("/auto")
-async def auto(request: Request):
-    body = await request.json()
-    worker.set_auto(bool(body.get("value", True)))
-    return JSONResponse(worker.status())
-
-
-@app.post("/bandwidth")
-async def bandwidth(request: Request):
-    body = await request.json()
-    worker.set_bandwidth(float(body.get("value", 0)))
-    return JSONResponse(worker.status())
-
-
-@app.get("/gpu")
-def gpu():
-    # GPU meters poll this on their own fast cadence. It only reads local
-    # nvidia-smi — no curator round-trip — so the util/VRAM bars stay live even
-    # when /status is slow waiting on the (sometimes busy) curator queue call.
-    g = read_gpu() or {}
-    us = worker.util_smooth()
-    if us is not None:
-        g["util_smooth"] = round(us, 1)   # autoscaler's EWMA — the UI bar tracks this
-    return JSONResponse(g)
-
-
-@app.get("/logs")
-def logs():
-    return JSONResponse({"lines": list(logbuf.LINES)})
-
-
-@app.get("/status")
-def status():
-    # Pure in-memory read: worker.status() is lock-free and the queue snapshot is
-    # kept fresh by a background poller — NO inline curator call, so this can't
-    # stall the status view when curator is buried under a big backlog.
-    worker.note_ui()          # a browser is watching → keep the queue snapshot warm
-    s = worker.status()
-    s["fc_url"] = cfg.fc_url
-    s["configured"] = bool(cfg.token)
-    s["queue"] = worker.latest_queue()
-    s["build"] = VERSION
-    return JSONResponse(s)
-
-
-_PAGE = """<!doctype html><html><head><meta charset=utf-8>
-<meta name=viewport content="width=device-width,initial-scale=1">
-<title>FabledCurator · GPU agent</title>
-<style>
- :root{--bg:#0f1216;--panel:#181c22;--panel2:#1e232b;--bd:#2a313b;--fg:#e9edf2;
-  --mut:#8b97a6;--acc:#e8923a;--grn:#46c46a;--red:#e8584d;--amb:#e8b23a}
- *{box-sizing:border-box}
- body{font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;margin:0;
-  background:radial-gradient(1200px 600px at 50% -10%,#1a2029,#0f1216);color:var(--fg)}
- .wrap{max-width:820px;margin:0 auto;padding:28px 20px 28px;height:100vh;
-  box-sizing:border-box;overflow:hidden;display:flex;flex-direction:column}
- header{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}
- .brand{display:flex;align-items:center;gap:10px;font-size:19px;font-weight:700;letter-spacing:.2px}
- .logo{color:var(--acc);font-size:20px}
- .brand .sub{color:var(--mut);font-weight:600;font-size:13px;text-transform:uppercase;letter-spacing:.12em}
- .conn{display:flex;align-items:center;gap:8px;color:var(--mut);font-size:13px;font-weight:600}
- .dot{width:9px;height:9px;border-radius:50%;background:var(--mut);box-shadow:0 0 0 0 rgba(0,0,0,0)}
- .dot.green{background:var(--grn);box-shadow:0 0 10px 1px rgba(70,196,106,.5)}
- .dot.amber{background:var(--amb)} .dot.red{background:var(--red)}
- .meta{color:var(--mut);margin:0 0 18px;font-size:13px}
- code{background:#11151a;border:1px solid var(--bd);padding:2px 7px;border-radius:6px;
-  font:12px ui-monospace,SFMono-Regular,Menlo,monospace;color:#cdd6e0}
- .card{background:linear-gradient(180deg,var(--panel),var(--panel2));border:1px solid var(--bd);
-  border-radius:14px;padding:16px 18px;margin-bottom:14px;box-shadow:0 1px 0 rgba(255,255,255,.02) inset}
- .card-h{font-size:11px;font-weight:800;letter-spacing:.12em;text-transform:uppercase;
-  color:var(--mut);margin-bottom:14px}
- .controls{display:flex;align-items:center;gap:10px;flex-wrap:wrap}
- .spacer{flex:1}
- .btn{font:600 14px system-ui;padding:.5rem 1rem;border:1px solid transparent;border-radius:9px;
-  cursor:pointer;color:#fff;transition:.12s}
- .btn:hover{transform:translateY(-1px)}
- .btn[disabled]{opacity:.45;pointer-events:none;transform:none}
- @keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
- .tile .n.busy{color:var(--acc);animation:pulse 1s ease-in-out infinite}
- .btn.start{background:linear-gradient(180deg,#2f9c4c,#247a3c)}
- .btn.stop{background:linear-gradient(180deg,#3a3f48,#2a2f37);color:#e9edf2;border-color:var(--bd)}
- .switch{display:inline-flex;align-items:center;gap:8px;cursor:pointer;font-weight:600;user-select:none}
- .switch input{display:none}
- .switch .track{width:38px;height:22px;border-radius:11px;background:#2a313b;position:relative;transition:.15s}
- .switch .track:after{content:"";position:absolute;top:2px;left:2px;width:18px;height:18px;border-radius:50%;
-  background:#cdd6e0;transition:.15s}
- .switch input:checked+.track{background:var(--acc)}
- .switch input:checked+.track:after{transform:translateX(16px);background:#fff}
- .stepper{display:inline-flex;align-items:center;gap:6px}
- .step{background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:8px;
-  width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
- .step:hover{border-color:var(--acc)}
- #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
-  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
- .unit{color:var(--mut);font-size:12px;font-weight:600}
- .hint{color:var(--mut);font-size:12px;margin-top:12px}
- .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
- .tile{background:#13171d;border:1px solid var(--bd);border-radius:10px;padding:12px 8px;text-align:center}
- .tile .n{font:800 22px ui-monospace,monospace;line-height:1.1}
- .tile .n.warn{color:var(--red)} .tile .n.ok{color:var(--grn)}
- .tile .l{font-size:10px;text-transform:uppercase;letter-spacing:.06em;color:var(--mut);margin-top:4px}
- .meters{display:flex;flex-direction:column;gap:10px;margin-bottom:14px}
- .meter-h{display:flex;justify-content:space-between;font-size:12px;color:var(--mut);margin-bottom:4px}
- .meter-h b{color:var(--fg);font-variant-numeric:tabular-nums}
- .bar{height:9px;border-radius:5px;background:#11151a;border:1px solid var(--bd);overflow:hidden}
- .bar>i{display:block;height:100%;width:0;background:linear-gradient(90deg,#3a7d57,var(--grn));transition:width .4s}
- #utilbar{background:linear-gradient(90deg,#9a5a1f,var(--acc))}
- #bufbar{background:linear-gradient(90deg,#2f5a9a,#4a86d8)}
- .queue{font:13px ui-monospace,monospace;color:var(--mut)}
- .banner{margin:0 0 14px;padding:.7rem .9rem;border-radius:10px;background:#3a2f12;
-  border:1px solid #5a4a17;color:#ffd98a;font-size:13px}
- .logs-h{display:flex;align-items:center;justify-content:space-between}
- .grow{flex:1;display:flex;flex-direction:column;min-height:0}
- .grow .logs{flex:1;min-height:0}
- .copybtn{font:600 11px system-ui;letter-spacing:.04em;text-transform:uppercase;
-  background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:7px;
-  padding:5px 11px;cursor:pointer}
- .copybtn:hover{border-color:var(--acc)}
- .logs{margin:0;background:#0b0e12;border:1px solid var(--bd);border-radius:10px;padding:12px;
-  overflow:auto;font:12px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;
-  color:#b9c4d0;white-space:pre-wrap;word-break:break-word}
-</style></head><body>
-<div class=wrap>
- <header>
-  <div class=brand><span class=logo>◆</span> FabledCurator <span class=sub>GPU agent</span></div>
-  <div class=conn><span class="dot" id=dot></span><span id=connlbl>—</span></div>
- </header>
- <p class=meta>Server <code id=fc>—</code> · token <code id=cfg>—</code> · build <code id=build>__BUILD__</code></p>
-
- <div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
-  a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
- </div>
- <div id=banner class=banner style=display:none>
-  curator unreachable — holding work + retrying, resumes on its own (no restart needed)
- </div>
-
- <section class=card>
-  <div class=card-h>Control</div>
-  <div class=controls>
-   <button class="btn start" id=startbtn onclick=act('start')>▶ Start</button>
-   <button class="btn stop" id=stopbtn onclick=act('stop')>■ Stop</button>
-   <div class=spacer></div>
-   <label class=switch><input type=checkbox id=autochk onchange="setauto(this.checked)"><span class=track></span>Auto</label>
-   <div class=stepper>
-    <button class=step onclick=setc(-1)>−</button>
-    <input id=conc type=number min=1 value=1 onchange="setv(this.value)">
-    <button class=step onclick=setc(1)>+</button>
-   </div>
-   <div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
-    <input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
-    <span class=unit>MB/s</span>
-   </div>
-  </div>
-  <div class=hint id=conchint>auto-tuning downloaders to keep the GPU fed · max 8</div>
- </section>
-
- <section class=card>
-  <div class=card-h>Status</div>
-  <div class=tiles>
-   <div class=tile><div class=n id=state>—</div><div class=l>state</div></div>
-   <div class=tile><div class=n id=jpm>—</div><div class=l>jobs / min</div></div>
-   <div class=tile><div class=n id=dpm>—</div><div class=l>downloads / min</div></div>
-   <div class=tile><div class="n ok" id=done>0</div><div class=l>processed</div></div>
-   <div class=tile><div class=n id=err>0</div><div class=l>errors</div></div>
-   <div class=tile><div class=n id=waited>0</div><div class=l>waited out</div></div>
-  </div>
-  <div class=meters>
-   <div class=meter><div class=meter-h><span>GPU util</span><b id=utillbl>—</b></div>
-    <div class=bar><i id=utilbar></i></div></div>
-   <div class=meter><div class=meter-h><span>VRAM</span><b id=vramlbl>—</b></div>
-    <div class=bar><i id=gpubar></i></div></div>
-   <div class=meter><div class=meter-h><span>buffer occupancy</span><b id=buflbl>—</b></div>
-    <div class=bar><i id=bufbar></i></div></div>
-  </div>
-  <div class=queue id=pipe>downloaders — · consumers — · on GPU 0</div>
-  <div class=queue id=queue>queue —</div>
- </section>
-
- <section class="card grow">
-  <div class="card-h logs-h">Logs
-   <button class=copybtn id=copybtn onclick=copyLogs()>Copy</button>
-  </div>
-  <pre class=logs id=logs>waiting for activity…</pre>
- </section>
-</div>
-<script>
- const PAGE_BUILD="__BUILD__"
- let CAP=8
- // Optimistic transitional state on click, then apply the POST's own status
- // response (it returns worker.status()) for instant feedback — don't wait on the
- // separate /status poll, which can lag behind the curator queue call.
- async function act(p){
-  pending(p==='start'?'starting':'stopping')
-  // Abort a slow POST after 8s so the buttons never stay stuck — the periodic
-  // /status refresh (now always fast) recovers the true state either way.
-  const ac=new AbortController(); const to=setTimeout(()=>ac.abort(),8000)
-  try{ applyStatus(await (await fetch('/'+p,{method:'POST',signal:ac.signal})).json()) }
-  catch{ refresh() /* on abort/error, repaint the real state from /status */ }
-  finally{ clearTimeout(to) }
- }
- function pending(label){
-  // Instant optimistic feedback on click; applyStatus (POST response, then the
-  // periodic poll) then owns the real state + which buttons are enabled.
-  state.textContent=label; state.className='n busy'
-  dot.className='dot amber'
-  startbtn.disabled=true; stopbtn.disabled=true
- }
- function setc(d){ if(conc.disabled)return; setv((parseInt(conc.value||'1'))+d) }
- async function setv(v){
-  v=Math.max(1,Math.min(CAP,parseInt(v)||1)); conc.value=v
-  await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
-   body:JSON.stringify({value:v})});refresh()
- }
- async function setauto(on){
-  await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
-   body:JSON.stringify({value:on})});refresh()
- }
- async function setbw(v){
-  v=Math.max(0,parseFloat(v)||0); bw.value=v
-  await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
-   body:JSON.stringify({value:v})});refresh()
- }
- async function refresh(){
-  let s; try{ s=await (await fetch('/status')).json() }catch{ return }
-  applyStatus(s)
- }
- function applyStatus(s){
-  // NB: don't write a separate `capn` element here — conchint.textContent below
-  // rewrites the whole hint (incl. the max), and any child element nested in it
-  // would be destroyed by that write, breaking the NEXT applyStatus call.
-  CAP=s.max_concurrency||8
-  // The backend owns the state now (stopped|starting|running|stopping) and drives
-  // every transition, so the pill is always truthful — no client-side guessing
-  // from active>0, which used to wedge on "stopping" forever.
-  const st=s.state||'stopped'
-  const running=st==='running'
-  const busy=(st==='starting'||st==='stopping')
-  // Stale-page guard: if the server is a newer build than this page, the cached
-  // controls may misbehave — tell the operator to reload.
-  if(s.build && s.build!==PAGE_BUILD) verbanner.style.display='block'
-  state.textContent=st
-  state.className='n'+(busy?' busy':'')
-  // Buttons follow the real state so you can't fight a transition: Start only
-  // from stopped; Stop only while up; both disabled through "stopping" until the
-  // backend truthfully lands on "stopped".
-  startbtn.disabled=(st!=='stopped')
-  stopbtn.disabled=!(running||st==='starting')
-  // Throughput rates arrive READY from the backend (jobs/min ≈ GPU throughput,
-  // dl/min ≈ fetch throughput), computed there on a fixed cadence — so they show
-  // a real number no matter how often this tab polls (a backgrounded tab throttles
-  // its timers, which used to leave a client-side delta-rate blank forever).
-  jpm.textContent=(s.jobs_per_min!=null)?Math.round(s.jobs_per_min):'—'
-  dpm.textContent=(s.downloads_per_min!=null)?Math.round(s.downloads_per_min):'—'
-  done.textContent=s.processed
-  err.textContent=s.errors; err.className='n'+(s.errors>0?' warn':'')
-  waited.textContent=s.transient||0
-  // Instantaneous pool state → demoted to the sub-line, where its jumpiness reads
-  // as live churn rather than a "broken" headline metric.
-  pipe.textContent='downloaders '+(s.downloaders!=null?s.downloaders:'—')+' · consumers '+(s.consumers!=null?s.consumers:'—')+' · on GPU '+(s.active||0)
-   +' · net '+(s.net_mb_s!=null?s.net_mb_s.toFixed(1):'—')+' MB/s'
-   +(s.bandwidth_limit_mb_s>0?(' / cap '+s.bandwidth_limit_mb_s):'')
-  if(document.activeElement!==bw && s.bandwidth_limit_mb_s!=null) bw.value=s.bandwidth_limit_mb_s
-  // Buffer occupancy bar (also driven here so it tracks the /status cadence).
-  if(s.buffer!=null && s.buffer_max){ const p=Math.round(100*s.buffer/s.buffer_max)
-   buflbl.textContent=s.buffer+' / '+s.buffer_max; bufbar.style.width=p+'%' }
-  // Auto on → dial reflects the auto-chosen count (read-only); off → manual.
-  if(document.activeElement!==autochk) autochk.checked=!!s.auto
-  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
-  conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
-   +(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)'
-     :(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':''))
-  if(document.activeElement!==conc) conc.value=s.concurrency
-  conc.max=CAP
-  // Connection pill + queue come only from the /status poll (the Start/Stop POST
-  // responses skip the slow curator call to stay snappy) — guard so an action
-  // response doesn't blank them.
-  if('configured' in s){
-   const ok=s.configured
-   fc.textContent=s.fc_url; cfg.textContent=ok?'set':'MISSING'
-   // Pill colour + label track the real state: green only when running AND
-   // curator is answering; amber for the transient states + a running-but-
-   // unreachable curator; grey when stopped; red with no token.
-   let dc='dot', lbl='stopped'
-   if(!ok){ dc='dot red'; lbl='no token' }
-   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
-   else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
-   else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
-   dot.className=dc; connlbl.textContent=lbl
-   banner.style.display=(st==='running' && !s.queue)?'block':'none'
-   queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
-  }
- }
- // GPU meters poll their OWN endpoint on a fast cadence — kept off /status so a
- // slow curator queue call can't freeze the bars (they only stale on refresh).
- let UAVG=null   // smoothed util for the bar (raw util swings 0↔99; show the trend)
- async function refreshGpu(){
-  let g; try{ g=await (await fetch('/gpu')).json() }catch{ return }
-  if(g && g.util_pct!=null){
-   // Prefer the agent's own EWMA (util_smooth) when running; otherwise smooth
-   // the raw reading here so a stopped agent's bar still glides, not jumps.
-   const raw=g.util_pct
-   UAVG = (g.util_smooth!=null) ? g.util_smooth
-         : (UAVG==null ? raw : 0.25*raw + 0.75*UAVG)
-   const used=g.mem_used_mb, tot=g.mem_total_mb||1
-   utillbl.textContent=Math.round(UAVG)+'% · '+g.temp_c+'°C'; utilbar.style.width=Math.round(UAVG)+'%'
-   vramlbl.textContent=used+' / '+tot+' MB'; gpubar.style.width=Math.round(100*used/tot)+'%'
-  } else { UAVG=null; utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
- }
- async function refreshLogs(){
-  try{
-   const r=await (await fetch('/logs')).json()
-   const el=logs, atBottom=el.scrollHeight-el.scrollTop-el.clientHeight<40
-   el.textContent=(r.lines&&r.lines.length)?r.lines.join('\\n'):'waiting for activity…'
-   if(atBottom) el.scrollTop=el.scrollHeight
-  }catch{}
- }
- async function copyLogs(){
-  const txt=logs.textContent||''
-  try{ await navigator.clipboard.writeText(txt) }
-  catch{ const t=document.createElement('textarea'); t.value=txt; document.body.appendChild(t);
-   t.select(); try{document.execCommand('copy')}catch{}; t.remove() }
-  copybtn.textContent='Copied'; setTimeout(()=>{copybtn.textContent='Copy'},1200)
- }
- refresh(); refreshGpu(); refreshLogs()
- setInterval(refresh,3000); setInterval(refreshGpu,1500); setInterval(refreshLogs,2500)
-</script></body></html>"""

agent/fc_agent/client.py

@@ -1,140 +0,0 @@
-"""HTTP client for the FabledCurator GPU-job API.
-
-The agent's ONLY contact with FC — lease/submit/heartbeat/fail + fetch image
-bytes, all over HTTP with the bearer token. No DB/Redis.
-"""
-import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
-
-
-class FcClient:
-    def __init__(self, base_url: str, token: str, agent_id: str):
-        self.base = base_url.rstrip("/")
-        self.agent_id = agent_id
-        # Main session: NO in-request retry — lease/fetch are cheap to redo and
-        # the worker loop already backs off + re-leases on failure. (Auto-retrying
-        # a lease could double-claim a batch if a response is lost.)
-        self.s = self._session(token)
-        # Submit session: retry in-place, because by submit time the GPU work is
-        # already DONE — a momentary blip (dropped connection, gateway 5xx during
-        # a curator redeploy) must not throw that work away and force a full
-        # re-download + recompute on another agent. A duplicate submit after a
-        # lost response is harmless: the job is already closed, so it just returns
-        # 409 lease_invalid (a no-op). Idempotent enough to retry POST safely.
-        retry = Retry(
-            total=3, connect=3, read=3, status=3,
-            backoff_factor=0.5,                       # ~0.5s, 1s, 2s between tries
-            status_forcelist=(500, 502, 503, 504),    # transient server/gateway
-            allowed_methods=frozenset({"POST"}),
-            raise_on_status=False,                     # let raise_for_status decide
-        )
-        self._submit_s = self._session(token, retry)
-
-    @staticmethod
-    def _session(token: str, retry: Retry | None = None) -> requests.Session:
-        s = requests.Session()
-        s.headers["Authorization"] = f"Bearer {token}"
-        # Many worker threads share a Session; the default pool (10) would
-        # throttle them + spam "connection pool is full". Size it for the cap.
-        adapter = HTTPAdapter(
-            pool_connections=64, pool_maxsize=64, max_retries=retry or 0
-        )
-        s.mount("http://", adapter)
-        s.mount("https://", adapter)
-        return s
-
-    def _submit(self, path: str, payload: dict) -> dict:
-        """POST to a submit endpoint on the RETRYING session (by submit time the
-        GPU work is done — a blip must not throw it away), raise on a hard error,
-        and return the parsed JSON. `agent_id` is added to every body."""
-        r = self._submit_s.post(
-            f"{self.base}{path}",
-            json={"agent_id": self.agent_id, **payload},
-            timeout=120,
-        )
-        r.raise_for_status()
-        return r.json()
-
-    def _post_quiet(self, path: str, payload: dict) -> None:
-        """Fire-and-forget POST on the main session — heartbeat/fail/release are
-        best-effort, so a transport error is swallowed (the worker's own retry and
-        the server's orphan-recovery cover a lost call). `agent_id` is added."""
-        try:
-            self.s.post(
-                f"{self.base}{path}",
-                json={"agent_id": self.agent_id, **payload},
-                timeout=30,
-            )
-        except requests.RequestException:
-            pass
-
-    def lease(self, batch_size: int) -> list[dict]:
-        r = self.s.post(
-            f"{self.base}/api/gpu/jobs/lease",
-            json={"agent_id": self.agent_id, "batch_size": batch_size},
-            timeout=30,
-        )
-        r.raise_for_status()
-        return r.json().get("jobs", [])
-
-    def submit(self, job_id: int, regions: list[dict], replace_kinds: list[str]) -> dict:
-        return self._submit("/api/gpu/jobs/submit", {
-            "job_id": job_id, "regions": regions, "replace_kinds": replace_kinds,
-        })
-
-    def submit_embedding(self, job_id: int, embedding: list, version: str) -> dict:
-        """Post a whole-image SigLIP embedding (the 'embed' task) → image_record."""
-        return self._submit("/api/gpu/jobs/submit_embedding", {
-            "job_id": job_id, "embedding": embedding, "embedding_version": version,
-        })
-
-    def heartbeat(self, job_ids: list[int]) -> None:
-        self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
-
-    def fail(self, job_id: int, error: str) -> None:
-        self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
-
-    def release(self, job_ids: list[int]) -> None:
-        # Graceful hand-back on stop so orphaned work is re-leased at once.
-        if not job_ids:
-            return
-        self._post_quiet("/api/gpu/jobs/release", {"job_ids": job_ids})
-
-    def fetch_image(self, image_url: str, throttle=None) -> bytes:
-        # image_url is a server-relative path ("/images/...").
-        # timeout=(connect, read): the read timeout is BETWEEN-BYTES, not total,
-        # so a large-but-flowing download still completes — but a stuck/dead
-        # connection (curator overloaded) fails in 60s instead of hanging a
-        # downloader for 180s and piling up concurrent stuck requests on curator.
-        # With a throttle (the worker's shared TokenBucket), the body is streamed
-        # in chunks and each chunk is charged to the global bandwidth budget —
-        # pausing between reads lets TCP flow control pace curator's send side.
-        with self.s.get(
-            f"{self.base}{image_url}", timeout=(10, 60), stream=throttle is not None
-        ) as r:
-            r.raise_for_status()
-            if throttle is None:
-                return r.content
-            buf = bytearray()
-            for chunk in r.iter_content(chunk_size=262_144):
-                throttle.take(len(chunk))
-                buf.extend(chunk)
-            return bytes(buf)
-
-    def is_reachable(self) -> bool:
-        """Cheap 'is curator responding at all right now?' check. Used to decide,
-        when a video can't be sampled, between a transient outage (keep retrying —
-        survives a redeploy) and an unprocessable file (fail it, don't loop)."""
-        try:
-            r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
-            return r.status_code < 500
-        except requests.RequestException:
-            return False
-
-    def queue_status(self) -> dict:
-        # Short timeout: this backs the UI /status poll, so a busy curator must
-        # not hang the page for long (the GPU meters poll /gpu separately).
-        r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
-        r.raise_for_status()
-        return r.json()

agent/fc_agent/config.py

@@ -1,90 +0,0 @@
-"""Agent config, all from env (the control container is configured at run)."""
-# Lazy annotations so the `from_env(cls) -> Config` self-reference is a string,
-# not evaluated at class-definition time — otherwise it NameErrors on the agent's
-# Python 3.10 (CI lints on 3.14, where PEP 649 hides this).
-from __future__ import annotations
-
-import os
-from dataclasses import dataclass
-
-
-def _bool_env(name: str, default: str = "") -> bool:
-    """A boolean env var — present + truthy ('1'/'true'/'yes') → True."""
-    return os.environ.get(name, default).lower() in ("1", "true", "yes")
-
-
-@dataclass
-class Config:
-    fc_url: str          # base URL of the FabledCurator web service
-    token: str           # the bearer token from Settings → Tagging → GPU agent
-    agent_id: str        # identifies this agent's leases
-    batch_size: int      # jobs a worker leases per round
-    concurrency: int     # INITIAL parallel workers (tunable live from the UI)
-    ccip_model: str      # imgutils CCIP model name ("" → imgutils default)
-    detector_level: str  # imgutils person-detector level: n|s|m|x
-    poll_idle_seconds: float   # wait between empty leases
-    embed_dtype: str     # torch dtype for the crop embedder: float16|float32
-    embed_model_override: str  # force a SigLIP-family model ("" → use the one
-    #                            the server announces in the lease)
-    auto_start: bool     # start the worker pool on boot (so a container restart
-    #                      resumes processing without anyone clicking Start)
-    auto_scale: bool     # autoscale the worker count (throughput hill-climb)
-    # Crop PROPOSERS (extra YOLO detectors that say where to crop). Each weight
-    # spec is an ultralytics name | http(s) URL | "hf_repo::file" ("" = off).
-    person_weights: str  # general COCO person detector (Western/realistic figs)
-    person_conf: float
-    anatomy_weights: str  # booru_yolo anime/furry/NSFW components
-    anatomy_conf: float
-    panel_weights: str    # comic-panel detector
-    panel_conf: float
-    max_components: int   # cap anatomy component crops per frame
-    max_panels: int       # cap panel crops per frame
-    max_figures: int      # cap figure boxes per frame (each = a CCIP call + crop)
-    max_regions: int      # hard cap on total regions per JOB (submit-size backstop)
-    dedupe_iou: float     # crops overlapping >= this (same kind) are near-dupes,
-    #                       dropped before the embed; >=1.0 disables it
-    frame_dedupe_distance: int  # video frames whose dHash differs by < this many
-    #                             bits are near-duplicates, dropped before detect;
-    #                             higher keeps more frames, 0 disables
-    ffmpeg_timeout: float  # hard ceiling (s) for ffmpeg-from-URL video sampling;
-    #                        generous so a SLOW media link still completes
-    bandwidth_limit_mb_s: float  # aggregate download cap in MEGABYTES/s across
-    #                              all downloaders + video streams (0 = unlimited);
-    #                              tunable live from the agent UI
-
-    @classmethod
-    def from_env(cls) -> Config:
-        return cls(
-            fc_url=os.environ.get("FC_URL", "http://localhost:8000").rstrip("/"),
-            token=os.environ.get("FC_TOKEN", ""),
-            agent_id=os.environ.get("AGENT_ID", "desktop-agent"),
-            batch_size=int(os.environ.get("BATCH_SIZE", "4")),
-            concurrency=int(os.environ.get("CONCURRENCY", "1")),
-            ccip_model=os.environ.get("CCIP_MODEL", ""),
-            detector_level=os.environ.get("DETECTOR_LEVEL", "m"),
-            poll_idle_seconds=float(os.environ.get("POLL_IDLE_SECONDS", "10")),
-            embed_dtype=os.environ.get("SIGLIP_DTYPE", "float16"),
-            embed_model_override=os.environ.get("EMBED_MODEL_NAME", ""),
-            auto_start=_bool_env("AUTO_START"),
-            auto_scale=_bool_env("AUTO_SCALE", "true"),
-            person_weights=os.environ.get("PERSON_WEIGHTS", "yolo11n.pt"),
-            person_conf=float(os.environ.get("PERSON_CONF", "0.35")),
-            anatomy_weights=os.environ.get("ANATOMY_WEIGHTS", ""),
-            anatomy_conf=float(os.environ.get("ANATOMY_CONF", "0.30")),
-            panel_weights=os.environ.get("PANEL_WEIGHTS", ""),
-            panel_conf=float(os.environ.get("PANEL_CONF", "0.30")),
-            max_components=int(os.environ.get("MAX_COMPONENTS", "8")),
-            max_panels=int(os.environ.get("MAX_PANELS", "8")),
-            max_figures=int(os.environ.get("MAX_FIGURES", "8")),
-            max_regions=int(os.environ.get("MAX_REGIONS", "128")),
-            dedupe_iou=float(os.environ.get("DEDUPE_IOU", "0.85")),
-            frame_dedupe_distance=int(os.environ.get("FRAME_DEDUPE_DISTANCE", "8")),
-            ffmpeg_timeout=float(os.environ.get("FFMPEG_TIMEOUT", "1200")),
-            # Default 8 MB/s (~64 Mbit/s): ~20% of the measured ~300 Mbit/s home
-            # WiFi, so browsing stays snappy while the agent works — yet MORE
-            # sweep throughput than the self-inflicted congestion collapse this
-            # replaces (2026-07-02: 8 unthrottled downloaders bufferbloated the
-            # link to ~1-1.5 MB/s per stream, browser included). Raise it (or 0)
-            # from the agent UI on wired/faster networks.
-            bandwidth_limit_mb_s=float(os.environ.get("BANDWIDTH_LIMIT_MB_S", "8")),
-        )

agent/fc_agent/crops.py

@@ -1,36 +0,0 @@
-"""Crop primitive — vendored from backend/app/services/ml/crops.py so the agent
-is self-contained. Keep in sync if the floor logic changes."""
-from PIL import Image
-
-MIN_CROP_FRACTION = 0.10
-MIN_CROP_PX = 64
-
-
-def crop_region(
-    img: Image.Image,
-    bbox: tuple[float, float, float, float],
-    *,
-    pad: float = 0.0,
-    min_fraction: float = MIN_CROP_FRACTION,
-    min_px: int = MIN_CROP_PX,
-) -> Image.Image | None:
-    """Crop a NORMALIZED bbox (x, y, w, h in [0,1]); None if below the size
-    floor (max of a fraction-of-short-side and an absolute pixel floor)."""
-    iw, ih = img.size
-    x, y, w, h = bbox
-    px, py, pw, ph = x * iw, y * ih, w * iw, h * ih
-    if pad:
-        px -= pw * pad / 2.0
-        py -= ph * pad / 2.0
-        pw *= (1.0 + pad)
-        ph *= (1.0 + pad)
-    left = max(0, int(round(px)))
-    top = max(0, int(round(py)))
-    right = min(iw, int(round(px + pw)))
-    bottom = min(ih, int(round(py + ph)))
-    if right <= left or bottom <= top:
-        return None
-    floor = max(min_px, int(min_fraction * min(iw, ih)))
-    if min(right - left, bottom - top) < floor:
-        return None
-    return img.crop((left, top, right, bottom)).convert("RGB")

agent/fc_agent/detectors.py

@@ -1,218 +0,0 @@
-"""Region PROPOSERS — small YOLO detectors that decide WHERE to crop. They run
-on the agent GPU and their boxes feed the crop → SigLIP → max-over-bag pipeline:
-
-  - person (general COCO yolo11n): full-figure boxes for realistic / Western art
-    the anime person-detector misses; NMS-merged with imgutils detect_person and
-    fed to CCIP (identity) + a concept crop.
-  - anatomy (booru_yolo): anime / furry / NSFW torso components (head, cat-head,
-    boob, hip, …) — concept crops aligned to the operator's tag vocabulary.
-  - panel (mosesb): a comic page → panel regions → concept crops.
-
-Each proposer is INDEPENDENTLY optional + guarded: a bad weight path or an
-inference error disables just that proposer (logged) and never breaks the
-worker, which still falls back to imgutils detection. Weights resolve from an
-ultralytics builtin name ("yolo11n.pt"), an http(s) URL, or "hf_repo::file" —
-cached under HF_HOME so the download happens once.
-"""
-import logging
-import os
-import threading
-import types
-from pathlib import Path
-
-log = logging.getLogger("fc_agent.detectors")
-_CACHE = Path(os.environ.get("HF_HOME", "/models")) / "yolo"
-
-
-def _resolve(spec: str) -> str | None:
-    """A local weights path (downloading if needed) or an ultralytics builtin
-    name. None if the spec is empty/unresolvable."""
-    if not spec:
-        return None
-    if "::" in spec:                                    # hf_repo::filename
-        repo, _, fname = spec.partition("::")
-        from huggingface_hub import hf_hub_download
-        return hf_hub_download(
-            repo_id=repo, filename=fname, cache_dir=str(_CACHE)
-        )
-    if spec.startswith(("http://", "https://")):
-        _CACHE.mkdir(parents=True, exist_ok=True)
-        dest = _CACHE / spec.rsplit("/", 1)[-1]
-        if not dest.is_file():
-            import requests
-            r = requests.get(spec, timeout=300)
-            r.raise_for_status()
-            dest.write_bytes(r.content)
-        return str(dest)
-    return spec                                         # ultralytics builtin name
-
-
-def _iou(a, b) -> float:
-    ax, ay, aw, ah = a
-    bx, by, bw, bh = b
-    ix = max(0.0, min(ax + aw, bx + bw) - max(ax, bx))
-    iy = max(0.0, min(ay + ah, by + bh) - max(ay, by))
-    inter = ix * iy
-    union = aw * ah + bw * bh - inter
-    return inter / union if union > 0 else 0.0
-
-
-def nms_merge(boxes, iou_thresh: float = 0.6):
-    """Greedy NMS over (bbox_norm, score, label) from possibly several detectors,
-    so the same figure found by two of them collapses to one (higher-score) box."""
-    kept = []
-    for bb, sc, lb in sorted(boxes, key=lambda b: b[1], reverse=True):
-        if all(_iou(bb, k[0]) < iou_thresh for k in kept):
-            kept.append((bb, sc, lb))
-    return kept
-
-
-def dedupe_crops(pending, iou_thresh: float = 0.85):
-    """Greedy high-IoU dedupe over a list of (crop, region_template) pairs, run
-    just before the batched SigLIP embed so we never embed the same region twice.
-
-    Figure boxes are already NMS-merged and each YOLO self-NMSes, but the combined
-    per-frame pile (figure→concept ∪ anatomy component→concept ∪ panel) can still
-    carry genuine near-duplicates across proposers — e.g. a figure box that nearly
-    coincides with an anatomy component on a solo bust, or overlapping booru head
-    classes on one head. Those embed the same region twice, wasting GPU and a slot
-    against max_regions.
-
-    Boxes are compared ONLY within the same output kind and dropped when they
-    overlap at >= iou_thresh, keeping the highest-scoring one. The HIGH default
-    threshold is deliberate: it collapses only true near-identical boxes while
-    preserving intentional nested crops across scopes (a whole figure vs a small
-    head component sit well below it) and distinct kinds (concept vs panel). A
-    value >= 1.0 effectively disables it (nothing but an exact box matches)."""
-    kept = []
-    kept_boxes: dict = {}   # kind -> [bbox, ...] already kept
-    for crop, tmpl in sorted(
-        pending, key=lambda p: p[1].get("score") or 0.0, reverse=True
-    ):
-        bb = tmpl.get("bbox")
-        prior = kept_boxes.setdefault(tmpl.get("kind"), [])
-        if bb is not None and any(_iou(bb, kb) >= iou_thresh for kb in prior):
-            continue
-        prior.append(bb)
-        kept.append((crop, tmpl))
-    return kept
-
-
-class YoloProposer:
-    """One lazily-loaded ultralytics YOLO. detect(image) → [(bbox_norm, score,
-    label)] with bbox normalized (x, y, w, h) in [0,1]. Self-disables on any
-    load/inference failure."""
-
-    def __init__(self, name, weights, conf=0.25, keep_labels=None):
-        self.name = name
-        self._spec = weights
-        self._conf = conf
-        self._keep = [k.lower() for k in keep_labels] if keep_labels else None
-        self._model = None
-        self._ok = True
-        self._lock = threading.Lock()
-
-    def _load(self):
-        if self._model is not None or not self._ok:
-            return
-        with self._lock:
-            if self._model is not None or not self._ok:
-                return
-            try:
-                from ultralytics import YOLO
-                path = _resolve(self._spec)
-                if path is None:
-                    self._ok = False
-                    return
-                self._model = YOLO(path)
-                # Disable ultralytics' load-time Conv+BN fusion. AutoBackend fuses
-                # the graph on the first predict; some checkpoints (yolo11n, the
-                # comic-panel model) crash that step with "'Conv' object has no
-                # attribute 'bn'" (a partially-fused / version-mismatched graph),
-                # which silently disabled those proposers (operator-flagged
-                # 2026-07-01). Unfused inference is correct — only marginally
-                # slower — and this is robust across ultralytics versions; if a
-                # future version ignores the override, the detect() guard below
-                # still self-disables the proposer instead of spamming per image.
-                inner = getattr(self._model, "model", None)
-                if inner is not None:
-                    inner.fuse = types.MethodType(lambda self, *a, **k: self, inner)
-                log.info("detector %s loaded (%s)", self.name, path)
-            except Exception as exc:   # noqa: BLE001
-                log.warning("detector %s disabled (load failed): %s", self.name, exc)
-                self._ok = False
-
-    def detect(self, image):
-        self._load()
-        if self._model is None:
-            return []
-        try:
-            res = self._model.predict(image, conf=self._conf, verbose=False)[0]
-        except Exception as exc:   # noqa: BLE001
-            # Permanently self-disable on the FIRST inference failure rather than
-            # re-throwing (and re-logging) on every image forever — an unfixable
-            # model fault degrades to "this proposer is off", logged once.
-            log.warning("detector %s disabled (inference failed): %s", self.name, exc)
-            self._ok = False
-            self._model = None
-            return []
-        iw, ih = image.size
-        names = getattr(res, "names", None) or {}
-        out = []
-        for b in res.boxes:
-            label = str(names.get(int(b.cls), int(b.cls))).lower()
-            if self._keep is not None and not any(k in label for k in self._keep):
-                continue
-            x0, y0, x1, y1 = (float(v) for v in b.xyxy[0].tolist())
-            out.append((
-                (x0 / iw, y0 / ih, (x1 - x0) / iw, (y1 - y0) / ih),
-                float(b.conf), label,
-            ))
-        return out
-
-
-class Proposers:
-    """The agent's proposer set, built from config. Each detector is optional —
-    an empty weight spec leaves that proposer off."""
-
-    def __init__(self, cfg):
-        self.cfg = cfg
-        self._person = (
-            YoloProposer("person-coco", cfg.person_weights,
-                         conf=cfg.person_conf, keep_labels=["person"])
-            if cfg.person_weights else None
-        )
-        self._anatomy = (
-            YoloProposer("anatomy", cfg.anatomy_weights, conf=cfg.anatomy_conf)
-            if cfg.anatomy_weights else None
-        )
-        self._panel = (
-            YoloProposer("panel", cfg.panel_weights, conf=cfg.panel_conf)
-            if cfg.panel_weights else None
-        )
-
-    def figures(self, image, base_boxes):
-        """Merge imgutils person boxes (base_boxes: [(bbox, score)]) with the
-        general COCO person detector → NMS'd figure boxes [(bbox, score, label)],
-        capped to the highest-scoring max_figures. Uncapped, a busy/huge image
-        (many characters) yields hundreds of boxes → hundreds of per-figure CCIP
-        calls + crops → a 30s+ job and an oversized submit (operator-flagged)."""
-        boxes = [(bb, sc if sc is not None else 1.0, "person") for bb, sc in base_boxes]
-        if self._person is not None:
-            boxes += self._person.detect(image)
-        return nms_merge(boxes)[: self.cfg.max_figures]   # nms_merge is score-desc
-
-    @staticmethod
-    def _top(detector, image, cap: int):
-        """Top-`cap` detections by score from an optional proposer (None → the
-        proposer is off → []). Shared by the anatomy + panel proposers, which
-        differ only in which detector and which cap."""
-        if detector is None:
-            return []
-        return sorted(detector.detect(image), key=lambda b: b[1], reverse=True)[:cap]
-
-    def components(self, image):
-        return self._top(self._anatomy, image, self.cfg.max_components)
-
-    def panels(self, image):
-        return self._top(self._panel, image, self.cfg.max_panels)

agent/fc_agent/embedder.py

@@ -1,77 +0,0 @@
-"""Crop EMBEDDER for the concept bag — model-agnostic (CLIP/SigLIP-family).
-
-The server trains its per-concept heads in the embedding space of whatever model
-its `embedder_model_version` names; a crop must be embedded with the SAME model
-or its vector lands in a different coordinate system and every head misfires. So
-the model identity (HF name + version) is ANNOUNCED BY THE SERVER in the lease —
-nothing here is hardcoded to SigLIP. Whatever name the server sends is loaded via
-transformers `get_image_features` (the CLIP/SigLIP-family image-tower call); a
-non-CLIP backbone (e.g. a DINO encoder) would need its own pooling adapter.
-
-torch on CUDA, fp16 by default to keep VRAM low on a shared desktop GPU — the
-tiny fp16-vs-fp32 difference is negligible for the linear heads (cosine ~0.999).
-A single inference lock serializes the forward pass: the pipeline is I/O-bound,
-so the GPU isn't the bottleneck, and one model shared across worker threads is
-safest behind a lock.
-"""
-import threading
-
-import numpy as np
-from PIL import Image
-
-
-class CropEmbedder:
-    def __init__(self, model_name: str, dtype: str = "float16"):
-        self._name = model_name
-        self._dtype_name = dtype
-        self._model = None
-        self._processor = None
-        self._torch = None
-        self._device = None
-        self._dt = None
-        self._load_lock = threading.Lock()
-        self._infer_lock = threading.Lock()
-
-    @property
-    def model_name(self) -> str:
-        return self._name
-
-    def load(self) -> None:
-        if self._model is not None:
-            return
-        with self._load_lock:
-            if self._model is not None:
-                return
-            import torch
-            from transformers import AutoImageProcessor, AutoModel
-
-            self._torch = torch
-            self._device = "cuda" if torch.cuda.is_available() else "cpu"
-            dt = getattr(torch, self._dtype_name, torch.float16)
-            if self._device == "cpu":
-                dt = torch.float32   # fp16 matmul is unsupported/slow on CPU
-            self._dt = dt
-            self._processor = AutoImageProcessor.from_pretrained(self._name)
-            model = AutoModel.from_pretrained(self._name, torch_dtype=dt)
-            model.eval().to(self._device)
-            self._model = model
-
-    def embed(self, image: Image.Image) -> list[float]:
-        """A crop → its embedding as a plain float list, ready to POST."""
-        return self.embed_batch([image])[0]
-
-    def embed_batch(self, images: list) -> list[list[float]]:
-        """Embed many crops in ONE forward pass — far better GPU utilisation +
-        only one lock acquisition than embedding each crop separately (which
-        starved the GPU and serialised the whole pool)."""
-        if not images:
-            return []
-        self.load()
-        torch = self._torch
-        enc = self._processor(images=images, return_tensors="pt")
-        pixel_values = enc["pixel_values"].to(self._device, self._dt)
-        with self._infer_lock, torch.no_grad():
-            out = self._model.get_image_features(pixel_values=pixel_values)
-        pooled = out.pooler_output if hasattr(out, "pooler_output") else out
-        arr = pooled.float().cpu().numpy().astype(np.float32)
-        return [row.reshape(-1).tolist() for row in arr]

agent/fc_agent/gpu.py

@@ -1,65 +0,0 @@
-"""GPU load readout via nvidia-smi (present in the container thanks to the
-NVIDIA Container Toolkit's `utility` capability). Returns None if unavailable —
-the UI just shows n/a (e.g. CPU-fallback run).
-
-Reads are CACHED and de-duplicated: the UI meter polls fast, /status reads it,
-and the autoscaler samples it — if each spawned its own `nvidia-smi` (slow on a
-busy GPU) those blocking subprocesses would pile up in the server's thread pool
-and make the Start/Stop buttons feel dead. So a short TTL serves recent callers
-from cache, and only ONE probe runs at a time (others get the last value)."""
-import subprocess
-import threading
-import time
-
-_TTL = 1.0                       # seconds a sample is reused before re-probing
-_lock = threading.Lock()
-_cache: dict | None = None
-_cache_t = 0.0
-_probing = False
-
-
-def _probe() -> dict | None:
-    try:
-        out = subprocess.run(
-            [
-                "nvidia-smi",
-                "--query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu",
-                "--format=csv,noheader,nounits",
-            ],
-            capture_output=True, text=True, timeout=5, check=True,
-        ).stdout.strip().splitlines()
-    except (OSError, subprocess.SubprocessError):
-        return None
-    if not out:
-        return None
-    parts = [p.strip() for p in out[0].split(",")]
-    try:
-        return {
-            "util_pct": int(float(parts[0])),
-            "mem_used_mb": int(float(parts[1])),
-            "mem_total_mb": int(float(parts[2])),
-            "temp_c": int(float(parts[3])),
-        }
-    except (ValueError, IndexError):
-        return None
-
-
-def read_gpu(max_age: float = _TTL) -> dict | None:
-    """Latest GPU reading, cached. Serves from cache when fresh; when stale,
-    exactly one caller re-probes while the rest get the last value — so request
-    threads never block behind more than one `nvidia-smi`."""
-    global _cache, _cache_t, _probing
-    now = time.monotonic()
-    with _lock:
-        fresh = _cache is not None and (now - _cache_t) < max_age
-        if fresh or _probing:        # fresh, or a probe is already running
-            return _cache
-        _probing = True
-    try:
-        val = _probe()
-    finally:
-        with _lock:
-            _cache = val
-            _cache_t = time.monotonic()
-            _probing = False
-    return val

agent/fc_agent/logbuf.py

@@ -1,44 +0,0 @@
-"""In-memory log ring buffer so the control UI can show recent agent logs
-(detector loads, job errors, autoscaler decisions, outage back-offs) without
-needing `docker logs`. A bounded deque holds the last N formatted lines; a
-logging.Handler appends to it; the UI polls /logs."""
-import logging
-from collections import deque
-
-LINES: deque[str] = deque(maxlen=400)
-
-
-class RingHandler(logging.Handler):
-    def emit(self, record: logging.LogRecord) -> None:
-        try:
-            LINES.append(self.format(record))
-        except Exception:
-            pass
-
-
-_installed = False
-
-
-def install(level: int = logging.INFO) -> None:
-    """Attach the ring handler to the root logger once. fc_agent module loggers
-    propagate to root, so their records land here."""
-    global _installed
-    if _installed:
-        return
-    _installed = True
-    h = RingHandler()
-    h.setFormatter(
-        logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%H:%M:%S")
-    )
-    root = logging.getLogger()
-    root.addHandler(h)
-    if root.level == logging.NOTSET or root.level > level:
-        root.setLevel(level)
-    # Keep the buffer signal-rich: silence the chatty HTTP/download libs (every
-    # HF model fetch logs per-request) so the console shows agent activity —
-    # detector loads, job errors, autoscale moves — not request spam.
-    for noisy in (
-        "uvicorn.access", "ultralytics", "httpx", "httpcore",
-        "huggingface_hub", "urllib3", "filelock",
-    ):
-        logging.getLogger(noisy).setLevel(logging.WARNING)

agent/fc_agent/media.py

@@ -1,253 +0,0 @@
-"""Image + video handling. Stills load directly; videos are sampled into frames
-(ffmpeg) at the cadence FC sends — so a video becomes a bag of per-frame
-instances, each with a timestamp."""
-import io
-import logging
-import os
-import signal
-import subprocess
-import tempfile
-import time
-
-from PIL import Image, ImageFile
-
-from .throttle import PidReadMeter
-
-log = logging.getLogger("fc_agent.media")
-
-# Load slightly-truncated images (a few missing trailing bytes) instead of
-# raising — matches the server embedder. These are common in scraped libraries
-# and would otherwise fail the job 3× then error (operator-flagged 2026-06-30).
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-# Disable PIL's decompression-bomb guard: this is a TRUSTED local library, not an
-# untrusted upload surface, so a legitimately huge image (high-res scans/prints,
-# 90M+ pixels) must load. The default 89M-pixel limit only WARNS, but PIL raises
-# DecompressionBombError at 2× (~179M px) — which would fail those jobs outright
-# (operator-flagged 2026-06-30, images of 90–95M px).
-Image.MAX_IMAGE_PIXELS = None
-
-
-def is_video(mime: str) -> bool:
-    return bool(mime) and (mime.startswith("video/") or mime in {"image/gif"})
-
-
-def _dhash(img: Image.Image, size: int = 8) -> int:
-    """Difference hash: compare adjacent pixels of a (size+1 × size) grayscale
-    thumbnail → a `size*size`-bit fingerprint. Cheap (64 comparisons on a 72-px
-    thumbnail) and robust to scaling/compression noise — near-identical frames
-    hash within a few bits, a real scene change moves many."""
-    small = img.convert("L").resize((size + 1, size))
-    px = list(small.getdata())
-    bits = 0
-    for row in range(size):
-        base = row * (size + 1)
-        for col in range(size):
-            bits = (bits << 1) | int(px[base + col] > px[base + col + 1])
-    return bits
-
-
-def dedupe_frames(
-    frames: list[tuple[float, Image.Image]], min_distance: int
-) -> list[tuple[float, Image.Image]]:
-    """Drop visually near-duplicate frames. A near-static video sampled into many
-    frames re-runs the WHOLE detect→CCIP→SigLIP chain on ~identical frames — the
-    dominant video load. Greedy perceptual-hash dedup: keep a frame only if its
-    dHash differs from every already-kept frame by >= min_distance bits (Hamming),
-    so a static run collapses to one frame while genuinely distinct scenes all
-    survive. Order + timestamps preserved. CPU-only (64-bit int XORs), so it runs
-    in the decode stage and spares the GPU the skipped frames entirely.
-
-    min_distance is the coarseness dial: higher keeps more frames (safer for brief
-    localized changes an 8×8 hash can miss), 0 disables. The first frame is always
-    kept (nothing to compare against)."""
-    if min_distance <= 0 or len(frames) <= 1:
-        return frames
-    kept: list[tuple[float, Image.Image]] = []
-    hashes: list[int] = []
-    for t, frame in frames:
-        h = _dhash(frame)
-        if all(bin(h ^ k).count("1") >= min_distance for k in hashes):
-            hashes.append(h)
-            kept.append((t, frame))
-    return kept
-
-
-def to_rgb(img: Image.Image) -> Image.Image:
-    """RGB, flattening any transparency onto white first. A naive convert('RGB')
-    on a palette-with-transparency image (common for character PNGs on a clear
-    background) lets PIL guess the transparent pixels — usually black artifacts
-    that bleed into the crop + the embedding (and the "should be converted to
-    RGBA" warning). Compositing over white gives a clean, consistent background."""
-    if img.mode in ("RGBA", "LA", "PA") or (
-        img.mode == "P" and "transparency" in img.info
-    ):
-        img = img.convert("RGBA")
-        bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
-        return Image.alpha_composite(bg, img).convert("RGB")
-    return img.convert("RGB")
-
-
-def load_image(data: bytes) -> Image.Image:
-    return to_rgb(Image.open(io.BytesIO(data)))
-
-
-# ffmpeg reconnect flags — resume a dropped HTTP transfer (a slow/contended media
-# store can cut a long stream) instead of failing the whole job. Relies only on
-# HTTP + Range, which every FC deployment serves → environment-agnostic.
-_RECONNECT = [
-    "-reconnect", "1", "-reconnect_streamed", "1",
-    "-reconnect_on_network_error", "1", "-reconnect_delay_max", "5",
-]
-
-
-def _collect_frames(
-    tmp: str, interval: float, cap: int
-) -> list[tuple[float, Image.Image]]:
-    out: list[tuple[float, Image.Image]] = []
-    names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
-    for i, name in enumerate(names[:cap]):
-        with Image.open(os.path.join(tmp, name)) as im:
-            out.append((round(i * interval, 2), to_rgb(im)))
-    return out
-
-
-def _terminate(proc: subprocess.Popen) -> None:
-    """Stop an ffmpeg cleanly, then hard-kill if it ignores SIGTERM."""
-    try:
-        # A bandwidth-paused (SIGSTOPped) process can't receive SIGTERM until it
-        # resumes — always CONT first so termination is prompt, not queued.
-        proc.send_signal(signal.SIGCONT)
-    except OSError:
-        pass
-    proc.terminate()
-    try:
-        proc.wait(timeout=2)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        try:
-            proc.wait(timeout=2)
-        except subprocess.TimeoutExpired:
-            pass
-
-
-def _pause(proc: subprocess.Popen, seconds: float, should_stop) -> bool:
-    """SIGSTOP ffmpeg for ~`seconds` of bandwidth debt, staying responsive to
-    Stop. While paused, the kernel socket buffer fills and TCP flow control
-    stalls curator's send side — that's the throttle. SIGCONT is ALWAYS sent
-    before returning. False = a Stop arrived mid-pause."""
-    try:
-        proc.send_signal(signal.SIGSTOP)
-    except OSError:
-        return True                     # already exited — nothing to pause
-    try:
-        end = time.monotonic() + seconds
-        while (left := end - time.monotonic()) > 0:
-            if should_stop and should_stop():
-                return False
-            time.sleep(min(0.5, left))
-        return True
-    finally:
-        try:
-            proc.send_signal(signal.SIGCONT)
-        except OSError:
-            pass
-
-
-def sample_frames_from_url(
-    url: str, interval_seconds: float, max_frames: int,
-    *, headers: str = "", timeout: float = 1200.0, should_stop=None,
-    governor=None,
-) -> tuple[list[tuple[float, Image.Image]], str | None]:
-    """Sample frames by pointing ffmpeg STRAIGHT at the media URL — it Range-reads
-    only the video index + up to max_frames worth of content, so the agent never
-    downloads the whole file (VR/4K originals run 800MB+ and would buffer ~1GB in
-    RAM and get cut off mid-download). Reconnect flags resume a dropped transfer;
-    the timeout is the per-video ceiling (a slow/reconnecting stream can otherwise
-    run for minutes). `should_stop` is polled while ffmpeg runs so a Stop KILLS the
-    subprocess at once — otherwise a downloader stuck in a long decode keeps the
-    agent "working" long after Stop. `governor` (the worker's shared TokenBucket)
-    meters ffmpeg's network reads from outside via /proc/<pid>/io and SIGSTOPs
-    the process into budget, so video streaming honors the same aggregate
-    bandwidth cap as still downloads.
-
-    Returns (frames, reason): frames is empty on failure/stop/timeout, and
-    `reason` then carries the SPECIFIC cause (ffmpeg's stderr tail / timeout) so
-    the caller can put it in the job's error — a bare "no frames" hid a filter
-    bug as "unprocessable" for weeks. None reason on success."""
-    interval = max(0.5, float(interval_seconds or 4.0))
-    cap = max(1, int(max_frames or 64))
-    hdr = ["-headers", headers] if headers else []
-    # select (NOT the fps filter): always keep the FIRST frame, then one per
-    # `interval` seconds of timestamp. fps=1/N emits round(duration/N) frames,
-    # which is ZERO for any clip shorter than ~N/2 seconds — a whole class of
-    # short animation loops failed as "unprocessable" that way (operator-flagged
-    # 2026-07-02: 0.5s/1.75s clips). scale=out_range=full converts limited-range
-    # yuv420p to full range so the mjpeg (jpg) encoder accepts it at default
-    # strictness instead of erroring on "non full-range YUV".
-    vf = (
-        f"select='isnan(prev_selected_t)+gte(t-prev_selected_t\\,{interval})',"
-        "scale=out_range=full"
-    )
-    with tempfile.TemporaryDirectory() as tmp:
-        pattern = os.path.join(tmp, "f_%05d.jpg")
-        cmd = ["ffmpeg", "-nostdin", "-loglevel", "error", *_RECONNECT, *hdr,
-               "-i", url, "-vf", vf, "-fps_mode", "vfr",
-               "-frames:v", str(cap), "-q:v", "3", pattern]
-        # ffmpeg's stderr goes to a file (not a PIPE, which could fill and
-        # deadlock; not DEVNULL, which is how a filter bug hid as "unprocessable"
-        # for weeks) — on failure its tail is logged so the operator can see WHY.
-        errpath = os.path.join(tmp, "stderr.txt")
-        try:
-            with open(errpath, "wb") as errf:
-                proc = subprocess.Popen(
-                    cmd, stdin=subprocess.DEVNULL,
-                    stdout=subprocess.DEVNULL, stderr=errf,
-                )
-                meter = PidReadMeter(proc.pid) if governor is not None else None
-                # Poll rather than block, so a Stop (or the per-video timeout) can
-                # kill a slow/wedged ffmpeg promptly instead of waiting it out.
-                start = time.monotonic()
-                while True:
-                    try:
-                        proc.wait(timeout=0.5)
-                        break
-                    except subprocess.TimeoutExpired:
-                        stopped = should_stop and should_stop()
-                        if stopped or (time.monotonic() - start > timeout):
-                            _terminate(proc)
-                            if stopped:
-                                return [], "stopped"
-                            log.warning("ffmpeg timed out after %.0fs: %s",
-                                        timeout, url)
-                            return [], f"ffmpeg timed out after {timeout:.0f}s"
-                        if meter is not None:
-                            read = meter.delta()
-                            if read is None:      # /proc gone → stop governing
-                                meter = None
-                            elif (debt := governor.charge(read)) > 0:
-                                # Over budget: pause ffmpeg until the bucket
-                                # recovers. Pause time counts toward `timeout`
-                                # (it stays the wedge backstop either way).
-                                if not _pause(proc, debt, should_stop):
-                                    _terminate(proc)
-                                    return [], "stopped"
-        except (OSError, ValueError) as exc:
-            return [], f"ffmpeg not runnable: {exc}"
-        frames = _collect_frames(tmp, interval, cap)
-        if not frames:
-            reason = f"ffmpeg exit {proc.returncode}: {_tail(errpath)}"
-            log.warning("ffmpeg produced no frames for %s — %s", url, reason)
-            return [], reason
-        return frames, None
-
-
-def _tail(path: str, limit: int = 300) -> str:
-    """Last `limit` chars of a (stderr) file, flattened — for failure logs."""
-    try:
-        with open(path, "rb") as f:
-            f.seek(0, os.SEEK_END)
-            f.seek(max(0, f.tell() - limit))
-            return f.read().decode("utf-8", "replace").replace("\n", " ").strip()
-    except OSError:
-        return "?"

agent/fc_agent/models.py

@@ -1,39 +0,0 @@
-"""imgutils model wrappers — the figure DETECTOR + the CCIP EMBEDDER.
-
-⚠️ VERIFY ON FIRST RUN: the exact imgutils function names/signatures + the CCIP
-model string can drift between dghs-imgutils releases. These are the two seams to
-check against your installed version (`pip show dghs-imgutils`):
-  - detect_person(image, level=...) -> [((x0,y0,x1,y1), label, score), ...]
-  - ccip_extract_feature(image, model=...) -> a vector (768-d for caformer)
-imgutils auto-downloads the ONNX models from HuggingFace on first use; GPU is
-used when onnxruntime-gpu is installed.
-"""
-import numpy as np
-from PIL import Image
-
-
-def detect_figures(image: Image.Image, level: str = "m") -> list[tuple[tuple, float | None]]:
-    """Person/figure bounding boxes, NORMALIZED (x, y, w, h in [0,1]) + score.
-    Returns [] if detection finds nothing (caller falls back to whole-image)."""
-    from imgutils.detect import detect_person
-
-    iw, ih = image.size
-    out = []
-    for (x0, y0, x1, y1), _label, score in detect_person(image, level=level):
-        out.append((
-            (x0 / iw, y0 / ih, (x1 - x0) / iw, (y1 - y0) / ih),
-            float(score),
-        ))
-    return out
-
-
-def ccip_vector(image: Image.Image, model: str | None = None) -> list[float]:
-    """The CCIP identity embedding of a (cropped) character image, as a plain
-    float list ready to POST."""
-    from imgutils.metrics import ccip_extract_feature
-
-    feat = (
-        ccip_extract_feature(image, model=model)
-        if model else ccip_extract_feature(image)
-    )
-    return np.asarray(feat, dtype=np.float32).reshape(-1).tolist()

agent/fc_agent/throttle.py

@@ -1,111 +0,0 @@
-"""Global download-bandwidth governor (one token bucket for the whole agent).
-
-The agent lives on someone's desktop and shares that desktop's network —
-typically WiFi, where saturating the link doesn't just slow other apps: it
-bufferbloats the airtime (RTT 21→45ms) and collapses EVERY connection,
-the operator's browser included. Measured 2026-07-02: the idle link moved
-~38 MB/s single-stream, but under the 8-downloader sweep every stream on the
-machine crawled at ~1-1.5 MB/s. So the cap is on the AGGREGATE, not per
-stream: still downloads pump their chunks through take(), and ffmpeg video
-streams — whose sockets live in a subprocess we can't wrap — are metered from
-outside via /proc/<pid>/io and paused (SIGSTOP) into budget using charge()'s
-debt signal; TCP flow control then stalls the sender while ffmpeg sleeps.
-
-Accounting is post-paid (charge the bytes first, then wait out any debt): the
-bytes have already crossed the network by the time we count them, and it means
-a chunk larger than one second of budget can never deadlock the bucket.
-Stdlib-only on purpose — unit-tested in CI, where the agent's ML deps
-don't exist.
-"""
-import threading
-import time
-
-
-class TokenBucket:
-    """Thread-safe token bucket in bytes/second. rate 0 = unlimited.
-
-    `consumed` is the monotonic total of bytes charged (throttled or not) —
-    the worker's rate loop derives the UI's "net MB/s" readout from it.
-    """
-
-    def __init__(self, rate_bytes_per_s: float = 0.0):
-        self._cond = threading.Condition()
-        self._rate = max(0.0, float(rate_bytes_per_s))
-        # Burst = one second of budget: enough that chunked reads stay smooth,
-        # small enough that a burst can't meaningfully lift the average.
-        self._level = self._rate
-        self._stamp = time.monotonic()
-        self.consumed = 0
-
-    @property
-    def rate(self) -> float:
-        return self._rate
-
-    def set_rate(self, rate_bytes_per_s: float) -> None:
-        """Retune live (the UI dial). Waiters re-check immediately, so raising
-        the cap (or lifting it with 0) unblocks a mid-download wait at once."""
-        with self._cond:
-            self._refill_locked()          # settle elapsed time at the OLD rate
-            self._rate = max(0.0, float(rate_bytes_per_s))
-            self._level = min(self._level, self._rate)
-            self._cond.notify_all()
-
-    def _refill_locked(self) -> None:
-        now = time.monotonic()
-        self._level = min(self._rate, self._level + (now - self._stamp) * self._rate)
-        self._stamp = now
-
-    def take(self, n: int) -> None:
-        """Charge n bytes and block until the budget recovers (stills path)."""
-        with self._cond:
-            self.consumed += n
-            if self._rate <= 0:
-                return
-            self._refill_locked()
-            self._level -= n
-            while self._level < 0:
-                # Wake early on set_rate; cap the wait so a big debt is paid in
-                # re-checked slices rather than one long uninterruptible sleep.
-                self._cond.wait(min(-self._level / self._rate, 0.5))
-                if self._rate <= 0:
-                    return
-                self._refill_locked()
-
-    def charge(self, n: int) -> float:
-        """Charge n bytes WITHOUT blocking; return seconds of debt (0 = within
-        budget). The ffmpeg governor can't block the subprocess's own reads, so
-        it SIGSTOPs the process for (about) the returned debt instead."""
-        with self._cond:
-            self.consumed += n
-            if self._rate <= 0:
-                return 0.0
-            self._refill_locked()
-            self._level -= n
-            return max(0.0, -self._level / self._rate)
-
-
-class PidReadMeter:
-    """Cumulative read-bytes meter for a subprocess, via /proc/<pid>/io.
-
-    `rchar` counts every read() syscall's bytes — for a streaming ffmpeg the
-    network reads dominate, so the delta is a good-enough aggregate-bandwidth
-    signal (it's a governor, not a billing meter). Returns None when /proc is
-    unavailable (process exited, or a non-Linux host): the caller then simply
-    doesn't govern — degrade to unthrottled rather than break video sampling.
-    """
-
-    def __init__(self, pid: int):
-        self._path = f"/proc/{pid}/io"
-        self._last = 0
-
-    def delta(self) -> int | None:
-        try:
-            with open(self._path, "rb") as f:
-                for line in f:
-                    if line.startswith(b"rchar:"):
-                        total = int(line.split()[1])
-                        d, self._last = total - self._last, total
-                        return max(0, d)
-        except (OSError, ValueError):
-            return None
-        return None

agent/requirements.txt

@@ -1,18 +0,0 @@
-# CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace).
-dghs-imgutils>=0.4
-# GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
-# server-side fallback run.
-onnxruntime-gpu
-# The crop EMBEDDER (concept bag). torch is installed separately in the
-# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic;
-# transformers loads whatever SigLIP-family model the server announces.
-transformers>=4.45
-# Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic
-# panel) that decide where to crop. Uses the torch already installed above.
-ultralytics>=8.3
-# Control surface + HTTP.
-fastapi
-uvicorn[standard]
-requests
-pillow
-numpy

agent/ruff.toml

@@ -1,7 +0,0 @@
-# The agent runs on the CUDA base image's Python 3.12 (Ubuntu 24.04) — NOT the
-# 3.14 that CI's ci-python image and the repo-root ruff.toml target. Pin the
-# agent to py312 so ruff enforces 3.12 compatibility and never auto-applies a
-# 3.14-only fix (e.g. unquoting a self-referential annotation, which PEP 649
-# makes safe on 3.14 but NameErrors on 3.12). Inherit the root lint rules.
-extend = "../ruff.toml"
-target-version = "py312"

alembic/env.py

@@ -1,28 +1,13 @@
 """Alembic environment — reads DATABASE_URL from app config."""
 
-import os
-import re
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config, pool, text
+from sqlalchemy import engine_from_config, pool
 
 from alembic import context
 from backend.app.config import get_config
 from backend.app.models import Base
 
-# Fail a blocked migration FAST instead of hanging forever. Migrations run
-# against the live DB while workers hold locks; 0040's `ALTER series_page` queued
-# behind a tag-merge that held a series_page lock for minutes (the merge runs an
-# unindexed full scan over image_record while repointing series_page) and hung
-# with no timeout — silent, indefinite (operator-flagged 2026-06-07). With a
-# lock_timeout a blocked DDL errors ("canceling statement due to lock timeout")
-# and the entrypoint's `alembic upgrade head` exits non-zero, so the deploy
-# retries / surfaces loudly rather than wedging. Override via env when a known
-# slow-lock window is expected.
-_MIGRATION_LOCK_TIMEOUT = os.environ.get("MIGRATION_LOCK_TIMEOUT", "30s")
-if not re.fullmatch(r"\d+\s*(ms|s|min)?", _MIGRATION_LOCK_TIMEOUT.strip()):
-    _MIGRATION_LOCK_TIMEOUT = "30s"  # ignore a malformed override
-
 config = context.config
 
 if config.config_file_name is not None:
@@ -53,15 +38,6 @@ def run_migrations_online() -> None:
         poolclass=pool.NullPool,
     )
     with connectable.connect() as connection:
-        # Session-level lock_timeout for every DDL statement in this run. Set
-        # (and commit) before alembic opens its own transaction so the GUC
-        # persists on this connection regardless of how alembic structures its
-        # transactions. Value is from our own env, so f-string interpolation is
-        # safe (and it's been pattern-validated above); SET takes no bind params.
-        connection.execute(
-            text(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'")
-        )
-        connection.commit()
         context.configure(
             connection=connection,
             target_metadata=target_metadata,

alembic/versions/0022_source_per_artist_platform.py

@@ -83,80 +83,55 @@ def upgrade() -> None:
         if not other_ids:
             continue
 
-        # STEP 2: PRE-merge ALL Posts with duplicate external_post_id
-        # across the entire (canonical + others) group, BEFORE the bulk
-        # reparent. Two cases must both be handled:
-        #   (A) canonical has Post X with epid=N; an "other" source has
-        #       Post Y with epid=N → after bulk UPDATE, (canonical, N)
-        #       collides with itself.
-        #   (B) two different "other" sources each have a Post with
-        #       epid=N; canonical has none → after bulk UPDATE, both
-        #       are repointed to (canonical, N) and the second collides.
-        # The earlier version of this migration only handled (A); the
-        # operator's deploy 2026-05-26 tripped (B) at line 139.
-        # Fix: group ALL Posts in the (artist, platform) by epid; for
-        # any group with count>1, pick the keep (prefer one already
-        # under canonical; else lowest id) and merge the rest into it.
-        all_posts = conn.execute(
+        # STEP 2: PRE-merge colliding Posts BEFORE the bulk reparent.
+        # Find pairs (keep, drop) where:
+        #   keep  = a Post under the canonical source
+        #   drop  = a Post under one of the non-canonical sources whose
+        #           external_post_id equals keep.external_post_id
+        # If we let the bulk UPDATE try to repoint drop onto canonical,
+        # uq_post_source_external_id fires row-by-row and aborts the
+        # whole migration.
+        colliding = conn.execute(
             text("""
-                SELECT external_post_id, id, source_id
-                FROM post
-                WHERE source_id = :canonical OR source_id = ANY(:others)
-                ORDER BY external_post_id, id
+                SELECT keep.id AS keep_id, drop_.id AS drop_id
+                FROM post AS keep
+                JOIN post AS drop_
+                  ON drop_.external_post_id = keep.external_post_id
+                 AND drop_.id != keep.id
+                WHERE keep.source_id = :canonical
+                  AND drop_.source_id = ANY(:others)
             """),
             {"canonical": canonical_id, "others": other_ids},
         ).fetchall()
-        by_epid: dict = {}
-        for epid, post_id, src_id in all_posts:
-            by_epid.setdefault(epid, []).append((post_id, src_id))
-        for _epid, posts in by_epid.items():
-            if len(posts) <= 1:
-                continue
-            # Prefer a Post already under canonical as the keep.
-            canonical_posts = [p for p in posts if p[1] == canonical_id]
-            if canonical_posts:
-                keep_id = canonical_posts[0][0]
-            else:
-                keep_id = posts[0][0]  # already sorted by id ASC
-            drop_ids = [p[0] for p in posts if p[0] != keep_id]
-            for drop_id in drop_ids:
-                # Pre-delete image_provenance rows under drop_ whose
-                # image_record_id ALREADY has a provenance under keep —
-                # the UPDATE below would otherwise repoint them and
-                # trip uq_image_provenance_image_post (alembic 0021)
-                # row-by-row before any after-the-fact dedupe could
-                # run. Operator's v26.05.26.3 deploy 2026-05-26 tripped
-                # this at line 123.
-                conn.execute(
-                    text("""
-                        DELETE FROM image_provenance
-                        WHERE post_id = :drop_
-                          AND image_record_id IN (
-                              SELECT image_record_id FROM image_provenance
-                              WHERE post_id = :keep
-                          )
-                    """),
-                    {"keep": keep_id, "drop_": drop_id},
-                )
-                # Now safe to repoint the survivors.
-                conn.execute(
-                    text("""
-                        UPDATE image_provenance SET post_id = :keep
-                        WHERE post_id = :drop_
-                    """),
-                    {"keep": keep_id, "drop_": drop_id},
-                )
-                conn.execute(
-                    text("""
-                        UPDATE image_record SET primary_post_id = :keep
-                        WHERE primary_post_id = :drop_
-                    """),
-                    {"keep": keep_id, "drop_": drop_id},
-                )
-                conn.execute(
-                    text("DELETE FROM post WHERE id = :drop_"),
-                    {"drop_": drop_id},
-                )
+        for keep_id, drop_id in colliding:
+            conn.execute(
+                text("""
+                    UPDATE image_provenance SET post_id = :keep
+                    WHERE post_id = :drop_
+                """),
+                {"keep": keep_id, "drop_": drop_id},
+            )
+            conn.execute(
+                text("""
+                    UPDATE image_record SET primary_post_id = :keep
+                    WHERE primary_post_id = :drop_
+                """),
+                {"keep": keep_id, "drop_": drop_id},
+            )
+            # Repointed provenance may now collide on
+            # uq_image_provenance_image_post (alembic 0021). Dedupe:
+            # keep min(id) per (image_record_id, post_id).
+            conn.execute(text("""
+                DELETE FROM image_provenance ip1
+                USING image_provenance ip2
+                WHERE ip1.image_record_id = ip2.image_record_id
+                  AND ip1.post_id = ip2.post_id
+                  AND ip1.id > ip2.id
+            """))
+            conn.execute(
+                text("DELETE FROM post WHERE id = :drop_"),
+                {"drop_": drop_id},
+            )
 
         # STEP 3: Bulk reparent the remaining Posts off the other
         # Sources. After step 2, no collisions on

alembic/versions/0023_drop_meta_rating_tag_kinds.py

@@ -1,99 +0,0 @@
-"""drop meta + rating tag kinds — operator-retired 2026-05-26
-
-Revision ID: 0023
-Revises: 0022
-Create Date: 2026-05-26
-
-Operator decided meta + rating aren't valid tag kinds for FC. Per-row
-behavior: DELETE existing rows (operator chose "clean break" over
-"convert to general"). All cascading FKs (image_tag, tag_alias,
-tag_allowlist, tag_reference_embedding, tag_suggestion_rejection,
-series_page) use ondelete="CASCADE" so a single DELETE on tag cleans
-the related rows in one go.
-
-After the data cleanup, recreate the tag_kind ENUM without 'meta' /
-'rating' (Postgres has no `ALTER TYPE ... DROP VALUE`; standard
-rename-create-cast-drop dance). The server default 'general' is
-dropped before the type swap and restored after.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0023"
-down_revision: Union[str, None] = "0022"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # 1. Delete tags of the retired kinds. CASCADE handles related tables.
-    op.execute("DELETE FROM tag WHERE kind IN ('meta', 'rating')")
-
-    # 2. Drop the CHECK constraint that references the enum's literal
-    #    values. Postgres can't resolve `kind = 'character'` across the
-    #    type swap below — the literal would bind to the new tag_kind
-    #    but the column is on tag_kind_old, producing
-    #    "operator does not exist: tag_kind = tag_kind_old".
-    #    (Operator-hit during the v26.05.26.5 deploy attempt; ck was
-    #    originally added by alembic 0002.) Recreated post-swap.
-    op.drop_constraint(
-        "ck_tag_fandom_requires_character", "tag", type_="check"
-    )
-
-    # 3. Drop the server default — ALTER COLUMN TYPE can't carry it
-    #    across the type swap below.
-    op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT")
-
-    # 4. Recreate the tag_kind enum without meta/rating.
-    op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old")
-    op.execute(
-        "CREATE TYPE tag_kind AS ENUM ("
-        "'artist', 'character', 'fandom', 'general', "
-        "'series', 'archive', 'post'"
-        ")"
-    )
-    op.execute(
-        "ALTER TABLE tag "
-        "ALTER COLUMN kind TYPE tag_kind "
-        "USING kind::text::tag_kind"
-    )
-    op.execute("DROP TYPE tag_kind_old")
-
-    # 5. Restore the server default.
-    op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'")
-
-    # 6. Restore the CHECK constraint (now bound to the new tag_kind).
-    op.create_check_constraint(
-        "ck_tag_fandom_requires_character",
-        "tag",
-        "(fandom_id IS NULL) OR (kind = 'character')",
-    )
-
-
-def downgrade() -> None:
-    # Add the values back to the enum so old code can boot. The deleted
-    # tag rows are gone permanently — no safe restore.
-    op.drop_constraint(
-        "ck_tag_fandom_requires_character", "tag", type_="check"
-    )
-    op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT")
-    op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old")
-    op.execute(
-        "CREATE TYPE tag_kind AS ENUM ("
-        "'artist', 'character', 'fandom', 'general', "
-        "'series', 'archive', 'post', 'meta', 'rating'"
-        ")"
-    )
-    op.execute(
-        "ALTER TABLE tag "
-        "ALTER COLUMN kind TYPE tag_kind "
-        "USING kind::text::tag_kind"
-    )
-    op.execute("DROP TYPE tag_kind_old")
-    op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'")
-    op.create_check_constraint(
-        "ck_tag_fandom_requires_character",
-        "tag",
-        "(fandom_id IS NULL) OR (kind = 'character')",
-    )

alembic/versions/0024_backfill_post_title_from_description.py

@@ -1,80 +0,0 @@
-"""backfill post.post_title from description first-line — 2026-05-27
-
-Revision ID: 0024
-Revises: 0023
-Create Date: 2026-05-27
-
-SubscribeStar gallery-dl always writes `title: ""` and embeds the leading
-sentence inside `content` HTML. FC's sidecar parser was leaving
-post_title NULL for every SubscribeStar post since FC-3 shipped. The
-parser fix (sidecar._first_line_text fallback) now synthesizes a title
-at parse time; this migration applies the same logic retroactively to
-existing rows.
-
-Operator-flagged 2026-05-27 after inspecting
-/mnt/Data/Patreon/Cheunart/subscribestar/ sidecars.
-
-Idempotent: only touches rows where post_title IS NULL or empty AND
-description IS NOT NULL. Re-running the migration is a no-op.
-"""
-from __future__ import annotations
-
-import re
-from typing import Sequence, Union
-
-from alembic import op
-from sqlalchemy import text
-
-revision: str = "0024"
-down_revision: Union[str, None] = "0023"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-_TAG_RE = re.compile(r"<[^>]+>")
-_WS_RE = re.compile(r"\s+")
-
-
-def _first_line_text(body: str, limit: int = 120) -> str | None:
-    """Mirror of sidecar._first_line_text. Kept inline so the migration
-    doesn't carry a runtime import dependency from app code that may
-    have moved by the time the migration is replayed years from now."""
-    if not body:
-        return None
-    text_ = _TAG_RE.sub(" ", body)
-    text_ = text_.replace("\xa0", " ")
-    for line in text_.splitlines():
-        line = _WS_RE.sub(" ", line).strip()
-        if line:
-            if len(line) > limit:
-                return line[: limit - 1].rstrip() + "…"
-            return line
-    return None
-
-
-def upgrade() -> None:
-    bind = op.get_bind()
-    rows = bind.execute(
-        text(
-            "SELECT id, description FROM post "
-            "WHERE (post_title IS NULL OR post_title = '') "
-            "AND description IS NOT NULL AND description <> ''"
-        )
-    ).fetchall()
-    updated = 0
-    for row in rows:
-        derived = _first_line_text(row.description)
-        if not derived:
-            continue
-        bind.execute(
-            text("UPDATE post SET post_title = :t WHERE id = :id"),
-            {"t": derived, "id": row.id},
-        )
-        updated += 1
-    print(f"0024: backfilled post_title on {updated} row(s)")
-
-
-def downgrade() -> None:
-    # No safe restore — we can't tell which post_titles were derived vs
-    # genuinely present. Leave the column alone on rollback.
-    pass

alembic/versions/0025_fix_subscribestar_post_ids.py

@@ -1,288 +0,0 @@
-"""sidecar-audit followup: correct external_post_id + post_url across all platforms
-
-Revision ID: 0025
-Revises: 0024
-Create Date: 2026-05-27
-
-Closes the operator-flagged 2026-05-27 sidecar audit findings. Three
-data-correctness bugs across non-Patreon platforms had been silently
-corrupting Posts since FC-3 shipped; the parser fix (sidecar.py, same
-commit) addresses new imports. This migration cleans up existing rows.
-
-Per-platform actions:
-
-  subscribestar — gallery-dl wrote the per-attachment id in `id` and
-    the actual post id in `post_id`. FC's parser picked `id`, so every
-    multi-image SubscribeStar post was fragmented into N Post rows.
-    1. For each SubscribeStar Post, read its sidecar (via the related
-       ImageRecord's on-disk path), pull `post_id`, overwrite
-       external_post_id and post_url.
-    2. Merge groups of Posts under one source that now share an
-       external_post_id (fragments of the same actual post). Same
-       ImageProvenance pre-delete + repoint dance as alembic 0022.
-
-  hentaifoundry — sidecars have NO `url` field; `src` is the image
-    URL. FC's parser stored post_url=NULL. Read each HF Post's sidecar
-    for `user` + `index`, derive the canonical /pictures/user/<u>/<i>
-    permalink. external_post_id (= `index`) was already correct.
-
-  discord — gallery-dl wrote the CDN attachment URL in `url`. FC's
-    parser stored that as post_url. Read each Discord Post's sidecar
-    for the server/channel/message triple, derive the proper
-    discord.com/channels/.../<message> permalink. external_post_id (=
-    `message_id`) was already correct.
-
-  pixiv — pure-SQL backfill: replace any `i.pximg.net`-style URL on
-    Post.post_url with the derived `/artworks/<id>` permalink. Pixiv
-    external_post_id (= `id`) was already correct; no sidecar IO
-    needed.
-
-Idempotent: re-running on already-corrected data is a no-op (skips
-rows whose derived value matches what's already stored).
-
-Posts whose related ImageRecord paths don't resolve on disk (orphaned
-filesystem state) are skipped with a count in the migration output —
-those will be picked up by a future deep-scan.
-"""
-from __future__ import annotations
-
-import json
-import re
-from pathlib import Path
-from typing import Sequence, Union
-
-from alembic import op
-from sqlalchemy import text
-
-revision: str = "0025"
-down_revision: Union[str, None] = "0024"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-# Mirror of sidecar._NUMBERING_PREFIX. Kept inline so the migration is
-# self-contained (the operator's banked rule:
-# reference_postgres_enum_swap_drop_checks.md says migrations shouldn't
-# import from runtime app code).
-_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$")
-
-
-def _find_sidecar(media_path: Path) -> Path | None:
-    """gallery-dl writes the sidecar under the unprefixed stem
-    (`HOLLOW-ICHIGO.json`) while the media file gets a NN_ ordering
-    prefix (`01_HOLLOW-ICHIGO.png`). Try in order:
-      1. <stem>.json next to the media
-      2. <media>.json next to the media (full-name variant)
-      3. strip the NN_ prefix from the stem, then <stripped>.json
-    """
-    if not media_path:
-        return None
-    cand = media_path.with_suffix(".json")
-    if cand.is_file():
-        return cand
-    cand = media_path.parent / f"{media_path.name}.json"
-    if cand.is_file():
-        return cand
-    m = _NUMBERING_PREFIX.match(media_path.stem)
-    if m:
-        cand = media_path.parent / f"{m.group(1)}.json"
-        if cand.is_file():
-            return cand
-    return None
-
-
-def _str_id(v) -> str | None:
-    """str() a JSON scalar id; reject bool (JSON booleans are ints in
-    Python's eyes but they aren't valid sidecar ids)."""
-    if isinstance(v, bool):
-        return None
-    if isinstance(v, (str, int)) and str(v).strip():
-        return str(v).strip()
-    return None
-
-
-def _str_field(v) -> str | None:
-    if isinstance(v, str) and v.strip():
-        return v.strip()
-    return None
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # ── PART 1: Per-platform corrections requiring filesystem IO ─────
-    # SubscribeStar, HentaiFoundry, Discord all need fields from the
-    # sidecar to construct the right post_url. We walk each Post's
-    # related ImageRecord.path to find the sidecar, read it, derive,
-    # and update.
-    targets = conn.execute(text("""
-        SELECT p.id, p.external_post_id, p.post_url, s.platform
-        FROM post p
-        JOIN source s ON s.id = p.source_id
-        WHERE s.platform IN ('subscribestar', 'hentaifoundry', 'discord')
-    """)).fetchall()
-
-    stats: dict[str, dict[str, int]] = {
-        plat: {"read": 0, "updated": 0, "no_sidecar": 0}
-        for plat in ("subscribestar", "hentaifoundry", "discord")
-    }
-    for post_row in targets:
-        plat = post_row.platform
-        path = _first_attachment_path(conn, post_row.id)
-        if not path:
-            stats[plat]["no_sidecar"] += 1
-            continue
-        sidecar = _find_sidecar(Path(path))
-        if sidecar is None:
-            stats[plat]["no_sidecar"] += 1
-            continue
-        try:
-            data = json.loads(sidecar.read_text(encoding="utf-8"))
-        except (OSError, json.JSONDecodeError):
-            stats[plat]["no_sidecar"] += 1
-            continue
-        stats[plat]["read"] += 1
-
-        new_epid = post_row.external_post_id
-        new_url = None
-        if plat == "subscribestar":
-            pid = _str_id(data.get("post_id"))
-            if pid:
-                new_epid = pid
-                new_url = f"https://www.subscribestar.com/posts/{pid}"
-        elif plat == "hentaifoundry":
-            user = _str_field(data.get("user")) or _str_field(data.get("artist"))
-            idx = _str_id(data.get("index"))
-            if user and idx:
-                new_url = f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}"
-        elif plat == "discord":
-            sid = _str_id(data.get("server_id"))
-            cid = _str_id(data.get("channel_id"))
-            mid = _str_id(data.get("message_id"))
-            if sid and cid and mid:
-                new_url = f"https://discord.com/channels/{sid}/{cid}/{mid}"
-
-        # Idempotent: skip if nothing changed.
-        if new_epid == post_row.external_post_id and new_url == post_row.post_url:
-            continue
-        conn.execute(
-            text("""
-                UPDATE post
-                SET external_post_id = :epid, post_url = :url
-                WHERE id = :id
-            """),
-            {"epid": new_epid, "url": new_url, "id": post_row.id},
-        )
-        stats[plat]["updated"] += 1
-
-    for plat, s in stats.items():
-        print(
-            f"0025: {plat} — read {s['read']} sidecars, "
-            f"updated {s['updated']} Posts, "
-            f"{s['no_sidecar']} Posts had no resolvable sidecar"
-        )
-
-    # ── PART 2: Merge SubscribeStar fragments now sharing epid ───────
-    # After Part 1, each group of Posts under one source with the SAME
-    # new external_post_id is a fragment-set of the same actual post.
-    # Merge to one canonical row. Pre-handle the same ImageProvenance
-    # collision pattern as alembic 0022 (uq_image_provenance_image_post).
-    fragment_groups = conn.execute(text("""
-        SELECT p.source_id, p.external_post_id,
-               ARRAY_AGG(p.id ORDER BY p.id ASC) AS post_ids
-        FROM post p
-        JOIN source s ON s.id = p.source_id
-        WHERE s.platform = 'subscribestar'
-          AND p.external_post_id IS NOT NULL
-        GROUP BY p.source_id, p.external_post_id
-        HAVING COUNT(*) > 1
-    """)).fetchall()
-
-    merged = 0
-    for grp in fragment_groups:
-        post_ids = list(grp.post_ids)
-        keep_id, *drop_ids = post_ids
-        for drop_id in drop_ids:
-            # Pre-DELETE colliding ImageProvenance under drop_ that
-            # already exist under keep (alembic 0022 banked the pattern).
-            conn.execute(
-                text("""
-                    DELETE FROM image_provenance
-                    WHERE post_id = :drop_
-                      AND image_record_id IN (
-                          SELECT image_record_id FROM image_provenance
-                          WHERE post_id = :keep
-                      )
-                """),
-                {"keep": keep_id, "drop_": drop_id},
-            )
-            conn.execute(
-                text("""
-                    UPDATE image_provenance SET post_id = :keep
-                    WHERE post_id = :drop_
-                """),
-                {"keep": keep_id, "drop_": drop_id},
-            )
-            conn.execute(
-                text("""
-                    UPDATE image_record SET primary_post_id = :keep
-                    WHERE primary_post_id = :drop_
-                """),
-                {"keep": keep_id, "drop_": drop_id},
-            )
-            conn.execute(
-                text("""
-                    UPDATE post_attachment SET post_id = :keep
-                    WHERE post_id = :drop_
-                """),
-                {"keep": keep_id, "drop_": drop_id},
-            )
-            conn.execute(
-                text("DELETE FROM post WHERE id = :drop_"),
-                {"drop_": drop_id},
-            )
-            merged += 1
-    print(f"0025: subscribestar — merged {merged} duplicate Post fragments")
-
-    # ── PART 3: Pixiv post_url backfill (pure SQL) ───────────────────
-    # Pixiv's external_post_id is already correct (gallery-dl's `id` is
-    # the post id). Only post_url needs derivation: replace anything
-    # under i.pximg.net (the file URL) with the /artworks/<id> permalink.
-    pixiv_updated = conn.execute(text("""
-        UPDATE post p
-        SET post_url = 'https://www.pixiv.net/artworks/' || p.external_post_id
-        FROM source s
-        WHERE p.source_id = s.id
-          AND s.platform = 'pixiv'
-          AND p.external_post_id IS NOT NULL
-          AND (p.post_url IS NULL
-               OR p.post_url LIKE 'https://i.pximg.net/%'
-               OR p.post_url LIKE 'http://i.pximg.net/%')
-    """)).rowcount
-    print(f"0025: pixiv — backfilled post_url on {pixiv_updated} Posts")
-
-
-def _first_attachment_path(conn, post_id: int) -> str | None:
-    """Return any ImageRecord.path attached to this post (via
-    ImageProvenance). Lowest-id row keeps the migration deterministic
-    so re-running on the same DB picks the same sidecar."""
-    row = conn.execute(
-        text("""
-            SELECT ir.path
-            FROM image_provenance ip
-            JOIN image_record ir ON ir.id = ip.image_record_id
-            WHERE ip.post_id = :pid
-            ORDER BY ip.id ASC
-            LIMIT 1
-        """),
-        {"pid": post_id},
-    ).first()
-    return row[0] if row else None
-
-
-def downgrade() -> None:
-    # Lossy: external_post_id values were overwritten with the correct
-    # post_id; original per-attachment ids weren't preserved. Post-merge
-    # also deleted drop rows. No safe restore. To roll back the schema
-    # invariant, fork from 0024 and re-run sidecar imports.
-    pass

alembic/versions/0026_import_task_recovery_count_refetched.py

@@ -1,53 +0,0 @@
-"""import_task.recovery_count + refetched — poison-pill circuit breaker
-
-Revision ID: 0026
-Revises: 0025
-Create Date: 2026-05-28
-
-Backs the import-task resilience work (operator-flagged 2026-05-28):
-
-- recovery_count: how many times recover_interrupted_tasks has
-  re-queued this row from a stuck 'processing' state. A row that
-  hard-crashes the worker (OOM / segfault on a corrupt or oversized
-  input) leaves no terminal flip, so the sweep re-queues it — and
-  without a cap it would loop forever, re-crashing the worker each
-  time. After MAX_RECOVERY_ATTEMPTS the sweep marks it 'failed' with a
-  diagnostic instead.
-
-- refetched: whether a one-shot re-download has already been attempted
-  for this task's file. Bounds the Layer-2 re-fetch remediation to a
-  single attempt so source-side corruption doesn't loop.
-
-Both default to 0 / false; additive, no backfill needed.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0026"
-down_revision: Union[str, None] = "0025"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "import_task",
-        sa.Column(
-            "recovery_count", sa.Integer(), nullable=False,
-            server_default="0",
-        ),
-    )
-    op.add_column(
-        "import_task",
-        sa.Column(
-            "refetched", sa.Boolean(), nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_task", "refetched")
-    op.drop_column("import_task", "recovery_count")

alembic/versions/0027_drop_migration_run.py

@@ -1,50 +0,0 @@
-"""drop migration_run — one-and-done GS/IR migration tooling removed
-
-Revision ID: 0027
-Revises: 0026
-Create Date: 2026-05-29
-
-The GS/IR migration tooling (services/migrators, /api/migrate, the
-run_migration task, LegacyMigrationCard, and the MigrationRun model) was
-removed after the migration cutover completed. This drops its now-orphaned
-run-log table. Downgrade recreates the table (mirrors the old model) so the
-migration is reversible.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects.postgresql import JSONB
-
-revision: str = "0027"
-down_revision: Union[str, None] = "0026"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_table("migration_run")
-
-
-def downgrade() -> None:
-    op.create_table(
-        "migration_run",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("kind", sa.String(length=32), nullable=False),
-        sa.Column("status", sa.String(length=32), nullable=False),
-        sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.false()),
-        sa.Column(
-            "started_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column(
-            "counts", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"),
-        ),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column(
-            "metadata", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"),
-        ),
-    )
-    op.create_index("ix_migration_run_kind", "migration_run", ["kind"])
-    op.create_index("ix_migration_run_status", "migration_run", ["status"])

alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py

@@ -1,190 +0,0 @@
-"""collapse-sidecar-synthetic: repoint Posts/ImageProvenance/DownloadEvents
-from `sidecar:<platform>:<slug>` synthetic Source anchors onto the real
-Source for the same (artist, platform) when one exists, then delete the
-synthetic.
-
-Revision ID: 0028
-Revises: 0027
-Create Date: 2026-05-31
-
-Background: alembic 0022 (2026-05-26) consolidated the old per-post-URL
-Source rows into one canonical Source per (artist, platform). When NO
-real campaign URL was salvageable among the candidates, it rewrote the
-canonical row to url='sidecar:<platform>:<slug>' enabled=false as a
-disabled anchor for any Posts already attached.
-
-That was fine while it was the only Source for that artist+platform.
-But: the unique constraint on Source is (artist_id, platform, url), not
-(artist_id, platform). When the operator later added the real
-subscription via the UI / extension / etc., a SECOND row landed —
-the real one — with id > the synthetic. Both coexisted.
-
-Two follow-on problems surfaced 2026-05-31:
-
-  1. The Subscriptions UI listed both rows. The synthetic was disabled
-     so the scheduler never polled it, but it looked like a phantom
-     subscription. (Fixed in same commit by SourceService.list filter.)
-  2. importer._source_for_sidecar picked Source by `ORDER BY id ASC
-     LIMIT 1`, so EVERY gallery-dl download since the real Source was
-     added attached its Post to the SYNTHETIC anchor, not the real
-     Source. (Fixed in same commit by preferring non-sidecar URLs.)
-
-This migration is the data half of the cleanup: for every (artist,
-platform) with both a synthetic AND a real Source, repoint the
-synthetic's children (Posts, ImageProvenance, DownloadEvents) onto the
-real Source and delete the synthetic. Reuses the same epid/provenance
-collision dance from alembic 0022 because the same uniqueness
-constraints fire row-by-row during bulk UPDATEs.
-
-Lone synthetic anchors — those where no real Source for the same
-(artist, platform) exists (e.g., filesystem-imported artist with no
-subscription added) — are LEFT INTACT. They anchor real imported
-content; deleting them would CASCADE-delete the Posts the operator
-imported. The SourceService.list filter hides them from the UI; the
-operator can delete them by hand if they want the underlying imports
-gone.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-from sqlalchemy import text
-
-revision: str = "0028"
-down_revision: Union[str, None] = "0027"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Find (artist_id, platform) groups where BOTH a sidecar synthetic
-    # and at least one real Source exist.
-    groups = conn.execute(text("""
-        SELECT artist_id, platform
-        FROM source
-        GROUP BY artist_id, platform
-        HAVING bool_or(url LIKE 'sidecar:%')
-           AND bool_or(url NOT LIKE 'sidecar:%')
-    """)).fetchall()
-
-    for artist_id, platform in groups:
-        rows = conn.execute(
-            text("""
-                SELECT id, url FROM source
-                WHERE artist_id = :a AND platform = :p
-                ORDER BY id ASC
-            """),
-            {"a": artist_id, "p": platform},
-        ).fetchall()
-
-        synthetic_ids = [sid for sid, url in rows if url.startswith("sidecar:")]
-        real_rows = [(sid, url) for sid, url in rows if not url.startswith("sidecar:")]
-        if not synthetic_ids or not real_rows:
-            continue  # belt+suspenders; the GROUP BY already filtered
-
-        # Canonical real: lowest-id non-sidecar Source.
-        canonical_id = real_rows[0][0]
-
-        # STEP A: PRE-merge Post collisions on (canonical, external_post_id).
-        # Mirror alembic 0022's pre-merge logic — when synth has Post X
-        # epid=N and real has Post Y epid=N, the bulk UPDATE below would
-        # trip uq_post_source_external_id row-by-row. Group all Posts
-        # under (canonical + synthetics) by epid; for any group >1,
-        # pick a keep (prefer one already under canonical, else lowest
-        # id) and merge the rest into it.
-        all_posts = conn.execute(
-            text("""
-                SELECT external_post_id, id, source_id
-                FROM post
-                WHERE source_id = :canonical OR source_id = ANY(:synths)
-                ORDER BY external_post_id, id
-            """),
-            {"canonical": canonical_id, "synths": synthetic_ids},
-        ).fetchall()
-        by_epid: dict = {}
-        for epid, post_id, src_id in all_posts:
-            by_epid.setdefault(epid, []).append((post_id, src_id))
-        for _epid, posts in by_epid.items():
-            if len(posts) <= 1:
-                continue
-            canonical_side = [p for p in posts if p[1] == canonical_id]
-            keep_id = canonical_side[0][0] if canonical_side else posts[0][0]
-            drop_ids = [p[0] for p in posts if p[0] != keep_id]
-            for drop_id in drop_ids:
-                # Pre-delete image_provenance rows under drop_ whose
-                # image_record_id already has provenance under keep —
-                # avoids tripping uq_image_provenance_image_post (0021)
-                # row-by-row during the repoint UPDATE.
-                conn.execute(
-                    text("""
-                        DELETE FROM image_provenance
-                        WHERE post_id = :drop_
-                          AND image_record_id IN (
-                              SELECT image_record_id FROM image_provenance
-                              WHERE post_id = :keep
-                          )
-                    """),
-                    {"keep": keep_id, "drop_": drop_id},
-                )
-                conn.execute(
-                    text("""
-                        UPDATE image_provenance SET post_id = :keep
-                        WHERE post_id = :drop_
-                    """),
-                    {"keep": keep_id, "drop_": drop_id},
-                )
-                conn.execute(
-                    text("""
-                        UPDATE image_record SET primary_post_id = :keep
-                        WHERE primary_post_id = :drop_
-                    """),
-                    {"keep": keep_id, "drop_": drop_id},
-                )
-                conn.execute(
-                    text("DELETE FROM post WHERE id = :drop_"),
-                    {"drop_": drop_id},
-                )
-
-        # STEP B: Bulk reparent the remaining Posts off the synthetics.
-        conn.execute(
-            text("""
-                UPDATE post SET source_id = :canonical
-                WHERE source_id = ANY(:synths)
-            """),
-            {"canonical": canonical_id, "synths": synthetic_ids},
-        )
-
-        # STEP C: Reparent ImageProvenance.source_id (denormalized FK;
-        # no UNIQUE on source_id, safe bulk).
-        conn.execute(
-            text("""
-                UPDATE image_provenance SET source_id = :canonical
-                WHERE source_id = ANY(:synths)
-            """),
-            {"canonical": canonical_id, "synths": synthetic_ids},
-        )
-
-        # STEP D: Reparent any DownloadEvent.source_id. Synthetics are
-        # enabled=false so the scheduler never created events for them;
-        # this is belt+suspenders for any rows planted by manual force
-        # or older code paths.
-        conn.execute(
-            text("""
-                UPDATE download_event SET source_id = :canonical
-                WHERE source_id = ANY(:synths)
-            """),
-            {"canonical": canonical_id, "synths": synthetic_ids},
-        )
-
-        # STEP E: Drop the now-empty synthetics.
-        conn.execute(
-            text("DELETE FROM source WHERE id = ANY(:synths)"),
-            {"synths": synthetic_ids},
-        )
-
-
-def downgrade() -> None:
-    # Lossy migration — synthetic Sources deleted, Posts repointed and
-    # potentially merged. No safe downgrade.
-    pass

alembic/versions/0029_drop_artist_copyright_ml_thresholds.py

@@ -1,71 +0,0 @@
-"""drop artist + copyright ml thresholds; lower general default to 0.50
-
-Revision ID: 0029
-Revises: 0028
-Create Date: 2026-06-01
-
-Operator-flagged 2026-06-01: the view modal's Suggestions panel hides
-most general-category predictions because the default threshold is
-0.95. Lowering the default to 0.50 (matches character) so general
-suggestions surface more aggressively; the value remains tunable in
-Settings → ML.
-
-Same change retires two ML suggestion categories whose Tag.kind
-surfaces are unused:
-
-- `artist`: retired in FC-2d-vii-c — artist identity is acquisition-
-  derived (image_record.artist_id), never ML-inferred. The threshold
-  column was a leftover from before that retirement.
-- `copyright`: retired 2026-06-01 — the app uses `fandom` for the
-  franchise/copyright concept (per TagsView.vue's doc comment); no
-  Tag rows of kind=copyright exist, and the threshold column never
-  fed anything user-visible.
-
-Both columns are dropped from ml_settings; the existing row's
-suggestion_threshold_general value is bumped from 0.95 to 0.50 iff
-it's still at the old default, so deployed installs pick up the new
-UX without overriding any operator tuning.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-from sqlalchemy import text
-
-revision: str = "0029"
-down_revision: Union[str, None] = "0028"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Bump the general threshold for installs still at the old default.
-    op.execute(text(
-        "UPDATE ml_settings "
-        "SET suggestion_threshold_general = 0.50 "
-        "WHERE id = 1 AND suggestion_threshold_general = 0.95"
-    ))
-    op.drop_column("ml_settings", "suggestion_threshold_artist")
-    op.drop_column("ml_settings", "suggestion_threshold_copyright")
-
-
-def downgrade() -> None:
-    # Restore the columns with their prior defaults. The bump from
-    # 0.95 → 0.50 isn't reversible without remembering whether the
-    # operator had explicitly set 0.95 (unlikely — that was just the
-    # default) so we leave the current general value as-is.
-    from sqlalchemy import Column, Float
-
-    op.add_column(
-        "ml_settings",
-        Column(
-            "suggestion_threshold_artist",
-            Float, nullable=False, server_default="0.30",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        Column(
-            "suggestion_threshold_copyright",
-            Float, nullable=False, server_default="0.50",
-        ),
-    )

alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py

@@ -1,145 +0,0 @@
-"""nullable post.source_id + denormalized post.artist_id; retire sidecar synthetics
-
-Revision ID: 0030
-Revises: 0029
-Create Date: 2026-06-01
-
-Operator-asked 2026-06-01 after the Dymkens orphan investigation: the
-sidecar synthetic Source pattern (`sidecar:<platform>:<slug>` rows
-with enabled=false) was technically correct but misled the operator
-into thinking they had phantom subscriptions. The synthetics existed
-solely to satisfy `Post.source_id NOT NULL` for filesystem-imported
-content with no real subscription.
-
-This migration makes the data model honest:
-
-1. **Post gets a denormalized `artist_id` column** so artist filters
-   work without traversing `Post → Source.artist_id`. Backfilled from
-   the existing Source linkage, then NOT NULL'd.
-2. **`Post.source_id` becomes nullable**, FK ondelete `CASCADE` → `SET
-   NULL`. Deleting a Source detaches its Posts instead of destroying
-   imported content (semantically: subscription ends, archive stays).
-3. **`ImageProvenance.source_id` becomes nullable** with the same FK
-   semantic change.
-4. **Sidecar synthetic Sources are deleted** — first NULL out the
-   FKs from Post + ImageProvenance pointing at them (so the implicit
-   CASCADE doesn't fire), then delete. DownloadEvent FK is unchanged
-   (still CASCADE'd, NOT NULL'd) — synthetics have `enabled=false`
-   so no events exist for them.
-
-Uniqueness handling: the existing `uq_post_source_external_id`
-(source_id, external_post_id) keeps working for source-bound Posts
-(Postgres treats NULL != NULL so NULL-source rows aren't deduped by
-it). A second partial unique index covers the NULL-source case on
-(artist_id, external_post_id) so filesystem-imported posts still
-dedupe within an artist.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy import text
-
-revision: str = "0030"
-down_revision: Union[str, None] = "0029"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Step 1: add Post.artist_id, initially nullable for backfill.
-    # FK naming follows the Base.metadata naming_convention
-    # (fk_<table>_<column>_<referred_table>) — alembic 0001 set this up.
-    op.add_column(
-        "post",
-        sa.Column("artist_id", sa.Integer, nullable=True),
-    )
-    op.create_foreign_key(
-        "fk_post_artist_id_artist", "post", "artist",
-        ["artist_id"], ["id"], ondelete="CASCADE",
-    )
-
-    # Step 2: backfill from Source.artist_id (every existing Post has a
-    # Source today, so every row gets populated).
-    conn.execute(text("""
-        UPDATE post p
-        SET artist_id = s.artist_id
-        FROM source s
-        WHERE p.source_id = s.id AND p.artist_id IS NULL
-    """))
-
-    # Sanity: count any remaining NULLs. Should be zero pre-this-migration.
-    remaining = conn.execute(text(
-        "SELECT COUNT(*) FROM post WHERE artist_id IS NULL"
-    )).scalar_one()
-    if remaining:
-        raise RuntimeError(
-            f"alembic 0030: {remaining} post rows have no resolvable "
-            f"artist_id after backfill. Investigate before continuing."
-        )
-
-    # Step 3: enforce NOT NULL + add index for artist-filter queries.
-    op.alter_column("post", "artist_id", nullable=False)
-    op.create_index("ix_post_artist_id", "post", ["artist_id"])
-
-    # Step 4: relax post.source_id + flip FK to SET NULL. The original FK
-    # name from alembic 0001 is `fk_post_source_id_source` per the
-    # NAMING_CONVENTION in models/base.py.
-    op.alter_column("post", "source_id", nullable=True)
-    op.drop_constraint("fk_post_source_id_source", "post", type_="foreignkey")
-    op.create_foreign_key(
-        "fk_post_source_id_source", "post", "source",
-        ["source_id"], ["id"], ondelete="SET NULL",
-    )
-
-    # Step 5: relax image_provenance.source_id + flip FK to SET NULL.
-    op.alter_column("image_provenance", "source_id", nullable=True)
-    op.drop_constraint(
-        "fk_image_provenance_source_id_source", "image_provenance",
-        type_="foreignkey",
-    )
-    op.create_foreign_key(
-        "fk_image_provenance_source_id_source", "image_provenance", "source",
-        ["source_id"], ["id"], ondelete="SET NULL",
-    )
-
-    # Step 6: partial unique index on (artist_id, external_post_id) for
-    # NULL-source Posts. The existing uq_post_source_external_id keeps
-    # guarding source-bound rows; NULL-source rows now dedupe within
-    # an artist.
-    op.execute(
-        "CREATE UNIQUE INDEX uq_post_artist_external_id_null_source "
-        "ON post (artist_id, external_post_id) "
-        "WHERE source_id IS NULL"
-    )
-
-    # Step 7: retire sidecar synthetic Sources. NULL out the references
-    # FIRST (the new FK is SET NULL so CASCADE wouldn't fire anyway, but
-    # being explicit makes the intent clear). Then delete the synthetic
-    # source rows. Any DownloadEvent rows under synthetics CASCADE-die
-    # with the source — synthetics have enabled=false so there shouldn't
-    # be any in practice.
-    conn.execute(text("""
-        UPDATE post
-        SET source_id = NULL
-        WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%')
-    """))
-    conn.execute(text("""
-        UPDATE image_provenance
-        SET source_id = NULL
-        WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%')
-    """))
-    deleted = conn.execute(text(
-        "DELETE FROM source WHERE url LIKE 'sidecar:%' RETURNING id"
-    )).rowcount
-    print(f"alembic 0030: deleted {deleted} sidecar synthetic source rows")
-
-
-def downgrade() -> None:
-    # Lossy migration — the deleted sidecar synthetics can't be
-    # restored from the orphan post.source_id / image_provenance.source_id
-    # values, and the partial unique index encodes a constraint that
-    # NULL-source Posts may now exist. No safe downgrade.
-    pass

alembic/versions/0031_source_backfill_runs_remaining.py

@@ -1,45 +0,0 @@
-"""source.backfill_runs_remaining: sticky deep-scan mode
-
-Revision ID: 0031
-Revises: 0030
-Create Date: 2026-06-01
-
-Tick vs backfill mode for subscription downloads. When
-`backfill_runs_remaining > 0`, the next N download runs use
-`skip: True` + 30-min timeout (walk full history). When 0, runs use
-`skip: "exit:20"` + 14.5-min timeout (catch-up mode, exits early once
-20 contiguous archived items are seen).
-
-Operator-flagged 2026-06-01 (Knuxy run #38887): a creator with ~550
-archived posts saturates the 870s catch-up timeout even when there is
-no new content, because gallery-dl's default `skip: True` keeps walking.
-Tick mode short-circuits that; backfill mode is the explicit opt-in for
-deep history scans.
-
-Default 0 (all existing subscriptions start in tick mode).
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0031"
-down_revision: Union[str, None] = "0030"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "source",
-        sa.Column(
-            "backfill_runs_remaining",
-            sa.Integer,
-            nullable=False,
-            server_default="0",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("source", "backfill_runs_remaining")

alembic/versions/0032_source_error_type.py

@@ -1,41 +0,0 @@
-"""source.error_type: surface ErrorType taxonomy in FailingSourcesCard
-
-Revision ID: 0032
-Revises: 0031
-Create Date: 2026-06-02
-
-Audit 2026-06-02: the backend computes 13 ErrorType categories (auth_error,
-rate_limited, not_found, access_denied, validation_failed, etc.) and
-stamps each one on DownloadEvent.metadata, but the Source row only carried
-the free-text last_error. Operators couldn't bulk-triage failing sources
-("all auth_error → rotate cookies, all rate_limited → just wait") without
-opening Logs per row.
-
-This column receives the last error_type from _update_source_health
-and gets cleared on a successful run. Nullable + indexed so the failing-
-sources rollup can filter/group cheaply.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0032"
-down_revision: Union[str, None] = "0031"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "source",
-        sa.Column("error_type", sa.String(length=32), nullable=True),
-    )
-    op.create_index(
-        "ix_source_error_type", "source", ["error_type"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_source_error_type", table_name="source")
-    op.drop_column("source", "error_type")

alembic/versions/0033_suggestion_threshold_default_070.py

@@ -1,48 +0,0 @@
-"""suggestion_threshold default 0.50 → 0.70
-
-Revision ID: 0033
-Revises: 0032
-Create Date: 2026-06-02
-
-Operator-flagged 2026-06-02 — the 0.50 default (set on 2026-06-01) is
-too noisy in practice; raise to 0.70 for both suggestion categories.
-
-Only conditionally updates singletons whose current value is still the
-2026-06-01 default (0.50). Operators who deliberately tuned their row
-to some other value (0.55, 0.65, 0.80, etc. via the Settings UI) keep
-their pick — the migration only catches the unchanged-default case.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0033"
-down_revision: Union[str, None] = "0032"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings "
-        "SET suggestion_threshold_character = 0.70 "
-        "WHERE id = 1 AND suggestion_threshold_character = 0.50"
-    )
-    op.execute(
-        "UPDATE ml_settings "
-        "SET suggestion_threshold_general = 0.70 "
-        "WHERE id = 1 AND suggestion_threshold_general = 0.50"
-    )
-
-
-def downgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings "
-        "SET suggestion_threshold_character = 0.50 "
-        "WHERE id = 1 AND suggestion_threshold_character = 0.70"
-    )
-    op.execute(
-        "UPDATE ml_settings "
-        "SET suggestion_threshold_general = 0.50 "
-        "WHERE id = 1 AND suggestion_threshold_general = 0.70"
-    )

alembic/versions/0034_artist_visit.py

@@ -1,53 +0,0 @@
-"""artist_visit: per-artist last-viewed timestamp for the "+N new" badge
-
-Revision ID: 0034
-Revises: 0033
-Create Date: 2026-06-03
-
-Powers the artists-directory "+N new since last visit" badge + ArtistView
-banner. Single row per artist (no user_id yet — rule #47 multi-user ACL
-is aspirational; widens to (user_id, artist_id) PK when User lands).
-
-Seed every existing artist with `last_viewed_at = NOW()` so the badge
-starts at 0 across the board — no noisy "you have 5000 unseen images"
-on first deploy. New artists auto-get a row via
-`ArtistService.find_or_create`.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0034"
-down_revision: Union[str, None] = "0033"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "artist_visit",
-        sa.Column(
-            "artist_id",
-            sa.Integer,
-            sa.ForeignKey("artist.id", ondelete="CASCADE"),
-            primary_key=True,
-        ),
-        sa.Column(
-            "last_viewed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-    )
-    # Seed: every existing artist starts "fully caught up". Without this,
-    # every operator with N artists would see N badges (worth of every
-    # image ever imported) on first deploy.
-    op.execute(
-        "INSERT INTO artist_visit (artist_id, last_viewed_at) "
-        "SELECT id, NOW() FROM artist"
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("artist_visit")

alembic/versions/0035_image_record_effective_date.py

@@ -1,70 +0,0 @@
-"""image_record.effective_date: materialized gallery sort key + index
-
-Revision ID: 0035
-Revises: 0034
-Create Date: 2026-06-04
-
-The gallery ordered/cursored on COALESCE(post.post_date,
-image_record.created_at) across the Post outer join. That expression spans
-two tables, so no index can serve it — every /scroll sorted a large slice
-of the library, and the frontend fired ten of them serially per initial
-load. Materialize the value into image_record.effective_date and index
-(effective_date DESC, id DESC) so the cursor scroll is an index range scan.
-
-Backfill = COALESCE(primary post's post_date, created_at) so existing rows
-keep their exact ordering. New rows get the created_at-equivalent server
-default; services/importer.py overrides it with the post's date when a
-primary post with a date is linked.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0035"
-down_revision: Union[str, None] = "0034"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Add nullable first so the backfill can populate before NOT NULL.
-    op.add_column(
-        "image_record",
-        sa.Column("effective_date", sa.DateTime(timezone=True), nullable=True),
-    )
-    # Pure set-based UPDATEs (no per-row params) — immune to the 65535
-    # bind-parameter ceiling regardless of library size.
-    op.execute(
-        """
-        UPDATE image_record AS ir
-        SET effective_date = COALESCE(p.post_date, ir.created_at)
-        FROM post AS p
-        WHERE ir.primary_post_id = p.id
-        """
-    )
-    op.execute(
-        """
-        UPDATE image_record
-        SET effective_date = created_at
-        WHERE effective_date IS NULL
-        """
-    )
-    op.alter_column(
-        "image_record",
-        "effective_date",
-        nullable=False,
-        server_default=sa.text("now()"),
-    )
-    # DESC/DESC matches the gallery's ORDER BY effective_date DESC, id DESC
-    # so the scroll is a forward index scan; raw SQL because alembic's
-    # column list doesn't express per-column DESC cleanly.
-    op.execute(
-        "CREATE INDEX ix_image_record_effective_date "
-        "ON image_record (effective_date DESC, id DESC)"
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_image_record_effective_date", table_name="image_record")
-    op.drop_column("image_record", "effective_date")

alembic/versions/0036_siglip_embedding_hnsw_index.py

@@ -1,41 +0,0 @@
-"""image_record.siglip_embedding: HNSW cosine index for "more like this"
-
-Revision ID: 0036
-Revises: 0035
-Create Date: 2026-06-04
-
-Gallery Phase 3 (visual similarity search) ranks images by
-`siglip_embedding.cosine_distance(source_embedding)`. Without an index that's
-a sequential scan computing a 1152-dim distance for every row — fine at small
-scale, but it grows linearly with the library. Add an HNSW index with
-`vector_cosine_ops` so the top-N nearest search is sub-50ms ANN.
-
-1152 dims is under pgvector's 2000-dim HNSW limit, so HNSW (no training,
-better recall than IVFFlat) is the right choice. ONE-TIME COST: building the
-index over the existing embeddings (~57k vectors on the operator's library)
-locks image_record for ~30-60s during this migration on deploy — acceptable
-for a single-operator homelab. NULL embeddings (videos / not-yet-embedded
-rows) are simply not indexed.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0036"
-down_revision: Union[str, None] = "0035"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Raw SQL: alembic's create_index doesn't express the `USING hnsw (...
-    # vector_cosine_ops)` access-method + opclass cleanly. Must match the
-    # query's cosine_distance operator class to be usable by the planner.
-    op.execute(
-        "CREATE INDEX ix_image_record_siglip_hnsw "
-        "ON image_record USING hnsw (siglip_embedding vector_cosine_ops)"
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_image_record_siglip_hnsw", table_name="image_record")

alembic/versions/0037_patreon_seen_media.py

@@ -1,53 +0,0 @@
-"""patreon_seen_media: per-source ledger of already-ingested Patreon media
-
-Revision ID: 0037
-Revises: 0036
-Create Date: 2026-06-05
-
-Native Patreon ingester (build step 2a). Replaces gallery-dl's
-archive.sqlite3 with our own queryable table. The downloader upserts one
-row per (source, media) so routine walks skip media we've already
-processed; a future "recovery" mode bypasses the ledger to re-walk.
-
-`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form
-``video:<post_id>:<media_id>`` — hence String(128). The unique
-constraint on (source_id, filehash) is the dedup upsert key.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0037"
-down_revision: Union[str, None] = "0036"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "patreon_seen_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("post_id", sa.String(64), nullable=True),
-        sa.Column(
-            "seen_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_patreon_seen_media_source_id"
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("patreon_seen_media")

alembic/versions/0038_patreon_failed_media.py

@@ -1,58 +0,0 @@
-"""patreon_failed_media: per-source dead-letter ledger for failing Patreon media
-
-Revision ID: 0038
-Revises: 0037
-Create Date: 2026-06-06
-
-Plan #705 (#7). Media that keeps failing to download/validate (404'd CDN,
-deleted post, geo-blocked Mux, persistently-corrupt bytes) gets recorded here
-with an attempt counter; once it crosses the dead-letter threshold the ingester
-skips it on routine walks (recovery still re-attempts). A clean download clears
-the row. UNIQUE (source_id, filehash) is the upsert key (same media key the
-seen-ledger uses).
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0038"
-down_revision: Union[str, None] = "0037"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "patreon_failed_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
-        sa.Column("last_error", sa.Text, nullable=True),
-        sa.Column(
-            "first_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.Column(
-            "last_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_patreon_failed_media_source_id"
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("patreon_failed_media")

alembic/versions/0039_library_audit_resume.py

@@ -1,40 +0,0 @@
-"""library_audit_run: resume cursor + progress timestamp for chunked scans
-
-Revision ID: 0039
-Revises: 0038
-Create Date: 2026-06-07
-
-scan_library_for_rule used to run one 2h pass that timed out on large libraries
-and monopolized the concurrency-1 maintenance queue (operator-flagged). It now
-runs short time-boxed chunks that re-enqueue: `resume_after_id` persists the
-keyset cursor so the next chunk continues where it left off, and
-`last_progress_at` lets the recovery sweep tell a progressing multi-chunk audit
-from a genuinely stuck one.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0039"
-down_revision: Union[str, None] = "0038"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "library_audit_run",
-        sa.Column(
-            "resume_after_id", sa.Integer, nullable=False, server_default="0"
-        ),
-    )
-    op.add_column(
-        "library_audit_run",
-        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("library_audit_run", "last_progress_at")
-    op.drop_column("library_audit_run", "resume_after_id")

alembic/versions/0040_series_chapters.py

@@ -1,108 +0,0 @@
-"""series chapters: chapter layer over series_page (FC-6.1)
-
-Revision ID: 0040
-Revises: 0039
-Create Date: 2026-06-07
-
-A series (Tag kind='series') gains an ordered chapter layer. Reading order
-becomes (series_chapter.chapter_number, series_page.page_number). Every existing
-series is backfilled into a single auto-chapter (chapter_number=1) holding its
-current flat pages, so no data is lost and the old flat ordering is preserved.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0040"
-down_revision: Union[str, None] = "0039"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "series_chapter",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "series_tag_id",
-            sa.Integer,
-            sa.ForeignKey("tag.id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column("chapter_number", sa.Integer, nullable=False),
-        sa.Column("title", sa.Text, nullable=True),
-        sa.Column(
-            "is_placeholder", sa.Boolean, nullable=False, server_default="false"
-        ),
-        sa.Column("stated_page_start", sa.Integer, nullable=True),
-        sa.Column("stated_page_end", sa.Integer, nullable=True),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-    )
-    op.create_index(
-        "ix_series_chapter_series_tag_id", "series_chapter", ["series_tag_id"]
-    )
-
-    # New columns on series_page; chapter_id starts nullable so we can backfill.
-    op.add_column(
-        "series_page", sa.Column("chapter_id", sa.Integer, nullable=True)
-    )
-    op.add_column(
-        "series_page", sa.Column("stated_page", sa.Integer, nullable=True)
-    )
-
-    conn = op.get_bind()
-    # One auto-chapter per existing series (any series_tag_id present in pages).
-    conn.execute(
-        sa.text(
-            "INSERT INTO series_chapter "
-            "(series_tag_id, chapter_number, is_placeholder, created_at, updated_at) "
-            "SELECT DISTINCT series_tag_id, 1, false, now(), now() "
-            "FROM series_page"
-        )
-    )
-    # Point every existing page at its series' auto-chapter.
-    conn.execute(
-        sa.text(
-            "UPDATE series_page sp "
-            "SET chapter_id = sc.id "
-            "FROM series_chapter sc "
-            "WHERE sc.series_tag_id = sp.series_tag_id"
-        )
-    )
-
-    # Now lock chapter_id down: NOT NULL + FK (cascade) + index.
-    op.alter_column("series_page", "chapter_id", nullable=False)
-    op.create_foreign_key(
-        "fk_series_page_chapter_id",
-        "series_page",
-        "series_chapter",
-        ["chapter_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-    op.create_index(
-        "ix_series_page_chapter_id", "series_page", ["chapter_id"]
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_series_page_chapter_id", table_name="series_page")
-    op.drop_constraint(
-        "fk_series_page_chapter_id", "series_page", type_="foreignkey"
-    )
-    op.drop_column("series_page", "stated_page")
-    op.drop_column("series_page", "chapter_id")
-    op.drop_index("ix_series_chapter_series_tag_id", table_name="series_chapter")
-    op.drop_table("series_chapter")

alembic/versions/0041_series_suggestions.py

@@ -1,98 +0,0 @@
-"""series suggestions: assisted-continuation matcher (FC-6.3)
-
-Revision ID: 0041
-Revises: 0040
-Create Date: 2026-06-07
-
-A confirm-only queue of "this post may continue this series" hints, plus two
-import_settings knobs (enable + score threshold) for the matcher.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0041"
-down_revision: Union[str, None] = "0040"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "series_suggestion",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "post_id",
-            sa.Integer,
-            sa.ForeignKey("post.id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column(
-            "series_tag_id",
-            sa.Integer,
-            sa.ForeignKey("tag.id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column("score", sa.Float, nullable=False),
-        sa.Column("signals", sa.JSON, nullable=True),
-        sa.Column(
-            "status", sa.String(16), nullable=False, server_default="pending"
-        ),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-        sa.UniqueConstraint(
-            "post_id", "series_tag_id", name="uq_series_suggestion_post_series"
-        ),
-    )
-    op.create_index(
-        "ix_series_suggestion_post_id", "series_suggestion", ["post_id"]
-    )
-    op.create_index(
-        "ix_series_suggestion_series_tag_id",
-        "series_suggestion",
-        ["series_tag_id"],
-    )
-    op.create_index(
-        "ix_series_suggestion_status", "series_suggestion", ["status"]
-    )
-
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "series_suggest_enabled",
-            sa.Boolean,
-            nullable=False,
-            server_default=sa.true(),
-        ),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "series_suggest_threshold",
-            sa.Float,
-            nullable=False,
-            server_default="0.5",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "series_suggest_threshold")
-    op.drop_column("import_settings", "series_suggest_enabled")
-    op.drop_index("ix_series_suggestion_status", table_name="series_suggestion")
-    op.drop_index(
-        "ix_series_suggestion_series_tag_id", table_name="series_suggestion"
-    )
-    op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion")
-    op.drop_table("series_suggestion")

alembic/versions/0042_series_chapter_stated_part.py

@@ -1,32 +0,0 @@
-"""series chapter stated_part: operator-facing Part N label (FC-6.4)
-
-Revision ID: 0042
-Revises: 0041
-Create Date: 2026-06-07
-
-A chapter's positional chapter_number is auto-managed (rewritten 1..N on
-reorder/delete), so it can't double as the installment number the operator wants
-to type (e.g. a series authored from a post that is Part 2). Add a nullable
-stated_part alongside it — the same split as series_page.page_number (order) vs
-series_page.stated_page (printed number). Nullable; the UI falls back to
-chapter_number when unset.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0042"
-down_revision: Union[str, None] = "0041"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "series_chapter", sa.Column("stated_part", sa.Integer, nullable=True)
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("series_chapter", "stated_part")

alembic/versions/0043_post_attachment_per_post_unique.py

@@ -1,62 +0,0 @@
-"""post_attachment: per-post sha uniqueness (empty-post flood fix)
-
-Revision ID: 0043
-Revises: 0042
-Create Date: 2026-06-08
-
-PostAttachment.sha256 was GLOBALLY unique, so a non-art file the creator attaches
-to many posts (a standard pdf/zip/link-card) only ever got ONE row — on the first
-post — leaving every later post a bare shell (no image, no attachment). The native
-Patreon backfill of Anduo surfaced 1589 such shells (operator-flagged 2026-06-08).
-
-Switch to PER-POST uniqueness: the on-disk blob stays sha-deduped, but each post
-gets its own row. Replace the unique sha256 index with a plain lookup index plus
-two partial uniques — (post_id, sha256) for real posts and (sha256) for the
-NULL-post filesystem case (still one row per file there).
-
-Existing data has ≤1 row per sha (the old global unique), so the new partial
-uniques can't be violated on upgrade — no data backfill needed here. The bare-post
-shells themselves are removed by the separate prune-empty-posts cleanup tool.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0043"
-down_revision: Union[str, None] = "0042"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Drop the global unique index; recreate it as a plain (non-unique) lookup
-    # index so sha-based reads keep their index (matches the model's index=True).
-    op.drop_index("ix_post_attachment_sha256", table_name="post_attachment")
-    op.create_index(
-        "ix_post_attachment_sha256", "post_attachment", ["sha256"],
-    )
-    op.create_index(
-        "uq_post_attachment_post_sha", "post_attachment",
-        ["post_id", "sha256"], unique=True,
-        postgresql_where=sa.text("post_id IS NOT NULL"),
-    )
-    op.create_index(
-        "uq_post_attachment_null_post_sha", "post_attachment",
-        ["sha256"], unique=True,
-        postgresql_where=sa.text("post_id IS NULL"),
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "uq_post_attachment_null_post_sha", table_name="post_attachment"
-    )
-    op.drop_index(
-        "uq_post_attachment_post_sha", table_name="post_attachment"
-    )
-    op.drop_index("ix_post_attachment_sha256", table_name="post_attachment")
-    op.create_index(
-        "ix_post_attachment_sha256", "post_attachment", ["sha256"],
-        unique=True,
-    )

alembic/versions/0044_ml_settings_tagger_store_floor.py

@@ -1,37 +0,0 @@
-"""ml_settings.tagger_store_floor
-
-The ingest confidence floor below which tagger predictions are not stored,
-promoted from the TAGGER_STORE_FLOOR env var to a DB-backed, UI-tunable
-setting. Default 0.70 (was an env default of 0.05): the suggestion path
-already filters at 0.70 and the centroid/learned path covers low-confidence
-preferred tags, so the sub-0.70 tail was redundant weight — it had grown
-image_record's TOAST to ~100 GB. See plan-task #764.
-
-Revision ID: 0044
-Revises: 0043
-Create Date: 2026-06-10
-
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0044"
-down_revision: Union[str, None] = "0043"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "tagger_store_floor", sa.Float(),
-            nullable=False, server_default="0.7",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "tagger_store_floor")

alembic/versions/0045_image_prediction_table.py

@@ -1,69 +0,0 @@
-"""image_prediction table (DDL only — backfill runs as a background task)
-
-Normalizes the per-image tagger predictions out of the JSON blob into a
-queryable table (#768). This migration creates ONLY the table + indexes — it
-is pure DDL and commits instantly, so web boots immediately.
-
-The data backfill from the existing image_record.tagger_predictions JSON is
-deliberately NOT done here. Doing it inline made the whole migration one
-transaction over the ~100 GB TOAST: nothing committed until the very end, it
-was invisible/unmonitorable mid-run, and an early MATERIALIZED-CTE form spilled
-the full 100 GB to temp. Instead the backfill is the
-backend.app.tasks.admin.backfill_image_predictions_task — batched by id window,
-committed per chunk (visible progress + resumable), idempotent
-(ON CONFLICT DO NOTHING). Trigger it from Settings → Maintenance once web is up.
-
-The old image_record.tagger_predictions column is left in place (vestigial) and
-dropped in a follow-up once the backfill + code cutover are verified — dropping
-it needs an ACCESS EXCLUSIVE lock on the hot image_record table (the 0044 lock
-class), so it's deferred to a quiesced-worker window.
-
-Revision ID: 0045
-Revises: 0044
-Create Date: 2026-06-10
-
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0045"
-down_revision: Union[str, None] = "0044"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "image_prediction",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "image_record_id", sa.Integer(),
-            sa.ForeignKey("image_record.id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column("raw_name", sa.String(length=255), nullable=False),
-        sa.Column("category", sa.String(length=64), nullable=False),
-        sa.Column("score", sa.Float(), nullable=False),
-        sa.UniqueConstraint(
-            "image_record_id", "raw_name", name="image_raw_name",
-        ),
-    )
-    op.create_index(
-        "ix_image_prediction_image", "image_prediction", ["image_record_id"],
-    )
-    op.create_index(
-        "ix_image_prediction_name_score", "image_prediction",
-        ["raw_name", "score"],
-    )
-    # No data backfill here — see the module docstring. The one-time copy from
-    # image_record.tagger_predictions runs as backfill_image_predictions_task
-    # (batched, resumable, idempotent), kept out of this transaction so web boots
-    # without waiting on a ~100 GB pass.
-
-
-def downgrade() -> None:
-    op.drop_index("ix_image_prediction_name_score", "image_prediction")
-    op.drop_index("ix_image_prediction_image", "image_prediction")
-    op.drop_table("image_prediction")

alembic/versions/0046_drop_tagger_predictions.py

@@ -1,43 +0,0 @@
-"""drop image_record.tagger_predictions (predictions normalized to image_prediction)
-
-Final step of #768. The per-tag predictions now live in the image_prediction
-table (backfilled from the JSON, read by suggestions + allowlist, written by
-tag_and_embed). The old JSON column is dead weight — and it's the ~100 GB of
-sub-0.70 score tail that bloated image_record's TOAST and broke DB backups
-(#739). Dropping it is a fast catalog change; it does NOT reclaim the disk on
-its own — run `VACUUM FULL image_record` (or pg_repack) afterward, off-hours,
-to return the space to the OS so backups go small.
-
-DROP COLUMN needs a brief ACCESS EXCLUSIVE lock on image_record; env.py's
-lock_timeout guards it, so quiesce the ml-worker if a tagging run is in flight
-(see the migration-lock reference). tagger_model_version is kept — it's the
-"has this been tagged / is it current?" signal the backfill sweep reads.
-
-Revision ID: 0046
-Revises: 0045
-Create Date: 2026-06-11
-
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0046"
-down_revision: Union[str, None] = "0045"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_column("image_record", "tagger_predictions")
-
-
-def downgrade() -> None:
-    # Re-add the column empty. The JSON data is not restored (it lived only in
-    # this column); a downgrade would re-tag or backfill from image_prediction
-    # separately if ever needed.
-    op.add_column(
-        "image_record",
-        sa.Column("tagger_predictions", sa.JSON(), nullable=True),
-    )

alembic/versions/0047_series_chapter_dividers.py

@@ -1,175 +0,0 @@
-"""series chapters become cosmetic dividers; pages become one series-global run
-
-FC-6.x reframe (#789). A series is now ONE flat, series-global ordered run of
-pages; chapters stop owning pages and become labeled dividers anchored to the
-page that begins them.
-
-Migration (order matters — series_page.chapter_id cascades, so it must be
-dropped BEFORE any chapter row is deleted, or pages would cascade away):
-  a. Renumber series_page.page_number to a series-global 1..N (ordered by the
-     OLD (chapter_number, page_number)).
-  b. Add series_chapter.anchor_page_id and populate it with each chapter's first
-     page (lowest new page_number).
-  c. Drop series_page.chapter_id (severs the cascade link).
-  d. Prune chapters that shouldn't become dividers: empty/placeholder ones (no
-     anchor) and the redundant unlabeled chapter that would sit at page 1.
-  e. Reshape series_chapter into the divider: drop chapter_number,
-     is_placeholder, stated_page_start/end; make anchor_page_id NOT NULL +
-     UNIQUE + FK→series_page ON DELETE CASCADE.
-
-Revision ID: 0047
-Revises: 0046
-Create Date: 2026-06-11
-
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0047"
-down_revision: Union[str, None] = "0046"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # a. series-global page numbering, preserving the old reading order.
-    op.execute(
-        """
-        WITH ordered AS (
-            SELECT sp.id,
-                   ROW_NUMBER() OVER (
-                       PARTITION BY sp.series_tag_id
-                       ORDER BY sc.chapter_number, sp.page_number, sp.id
-                   ) AS rn
-            FROM series_page sp
-            JOIN series_chapter sc ON sc.id = sp.chapter_id
-        )
-        UPDATE series_page sp
-        SET page_number = ordered.rn
-        FROM ordered
-        WHERE sp.id = ordered.id
-        """
-    )
-
-    # b. anchor each existing chapter at its first page (lowest new page_number).
-    op.add_column(
-        "series_chapter",
-        sa.Column("anchor_page_id", sa.Integer(), nullable=True),
-    )
-    op.execute(
-        """
-        WITH firsts AS (
-            SELECT DISTINCT ON (sp.chapter_id)
-                   sp.chapter_id, sp.id AS page_id
-            FROM series_page sp
-            ORDER BY sp.chapter_id, sp.page_number, sp.id
-        )
-        UPDATE series_chapter sc
-        SET anchor_page_id = firsts.page_id
-        FROM firsts
-        WHERE firsts.chapter_id = sc.id
-        """
-    )
-
-    # c. sever the ownership link (drops the FK + index with the column) BEFORE
-    #    pruning chapters, so deleting a chapter can't cascade-delete its pages.
-    op.drop_column("series_page", "chapter_id")
-
-    # d. prune chapters that don't become dividers: placeholders / empty ones
-    #    (no anchor), and the unlabeled chapter that would land redundantly at
-    #    page 1 (the series just starts — no divider needed there).
-    op.execute(
-        """
-        DELETE FROM series_chapter sc
-        USING (
-            SELECT sc2.id
-            FROM series_chapter sc2
-            LEFT JOIN series_page sp ON sp.id = sc2.anchor_page_id
-            WHERE sc2.anchor_page_id IS NULL
-               OR (sp.page_number = 1
-                   AND sc2.title IS NULL
-                   AND sc2.stated_part IS NULL)
-        ) gone
-        WHERE sc.id = gone.id
-        """
-    )
-
-    # e. reshape into the divider model.
-    op.drop_column("series_chapter", "chapter_number")
-    op.drop_column("series_chapter", "is_placeholder")
-    op.drop_column("series_chapter", "stated_page_start")
-    op.drop_column("series_chapter", "stated_page_end")
-    op.alter_column("series_chapter", "anchor_page_id", nullable=False)
-    op.create_unique_constraint(
-        "uq_series_chapter_anchor_page", "series_chapter", ["anchor_page_id"]
-    )
-    op.create_foreign_key(
-        "fk_series_chapter_anchor_page",
-        "series_chapter",
-        "series_page",
-        ["anchor_page_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-
-
-def downgrade() -> None:
-    # Lossy: dividers can't be reconstructed as owning chapters. Collapse back to
-    # exactly one chapter per series that owns all its pages in order.
-    op.add_column(
-        "series_page", sa.Column("chapter_id", sa.Integer(), nullable=True)
-    )
-    op.drop_constraint(
-        "fk_series_chapter_anchor_page", "series_chapter", type_="foreignkey"
-    )
-    op.drop_constraint(
-        "uq_series_chapter_anchor_page", "series_chapter", type_="unique"
-    )
-    op.drop_column("series_chapter", "anchor_page_id")
-    op.add_column(
-        "series_chapter",
-        sa.Column(
-            "chapter_number", sa.Integer(), nullable=False, server_default="1"
-        ),
-    )
-    op.add_column(
-        "series_chapter",
-        sa.Column(
-            "is_placeholder", sa.Boolean(), nullable=False,
-            server_default="false",
-        ),
-    )
-    op.add_column(
-        "series_chapter",
-        sa.Column("stated_page_start", sa.Integer(), nullable=True),
-    )
-    op.add_column(
-        "series_chapter",
-        sa.Column("stated_page_end", sa.Integer(), nullable=True),
-    )
-    op.execute("DELETE FROM series_chapter")
-    op.execute(
-        """
-        INSERT INTO series_chapter (series_tag_id, chapter_number)
-        SELECT DISTINCT series_tag_id, 1 FROM series_page
-        """
-    )
-    op.execute(
-        """
-        UPDATE series_page sp
-        SET chapter_id = sc.id
-        FROM series_chapter sc
-        WHERE sc.series_tag_id = sp.series_tag_id
-        """
-    )
-    op.alter_column("series_page", "chapter_id", nullable=False)
-    op.create_foreign_key(
-        "fk_series_page_chapter",
-        "series_page",
-        "series_chapter",
-        ["chapter_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )

alembic/versions/0048_series_page_pending_status.py

@@ -1,45 +0,0 @@
-"""series_page pending staging: status + nullable page_number (#789 Phase 2)
-
-Pages added from a post no longer append straight into the run — they land
-'pending' with a NULL page_number, staged grouped by their source post so the
-operator can drop junk (text-free alts, bumpers) and place the keepers into the
-sequence. A page only gets a series-global page_number once it's 'placed'.
-
-Revision ID: 0048
-Revises: 0047
-Create Date: 2026-06-11
-
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0048"
-down_revision: Union[str, None] = "0047"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "series_page",
-        sa.Column(
-            "status", sa.String(length=16), nullable=False,
-            server_default="placed",
-        ),
-    )
-    op.alter_column(
-        "series_page", "page_number",
-        existing_type=sa.Integer(), nullable=True,
-    )
-
-
-def downgrade() -> None:
-    # Lossy: pending pages are unsorted staging rows with no order — drop them.
-    op.execute("DELETE FROM series_page WHERE status = 'pending'")
-    op.alter_column(
-        "series_page", "page_number",
-        existing_type=sa.Integer(), nullable=False,
-    )
-    op.drop_column("series_page", "status")

alembic/versions/0049_external_link_table.py

@@ -1,90 +0,0 @@
-"""external_link table — off-platform file-host links found in post bodies
-
-Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox /
-Pixeldrain and link them in the post text. This table records each such link
-(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger
-the download worker (a later slice) walks. `url` keeps the FULL link including
-the `#fragment` — mega.nz's decryption key lives there; truncating it makes the
-file undownloadable.
-
-CHECK whitelists for host + status include the full enum up front (incl. the
-download-worker statuses) so the worker slice needs no constraint migration.
-
-Revision ID: 0049
-Revises: 0048
-Create Date: 2026-06-14
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0049"
-down_revision: Union[str, None] = "0048"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "external_link",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "post_id", sa.Integer(),
-            sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False,
-        ),
-        sa.Column(
-            "artist_id", sa.Integer(),
-            sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True,
-        ),
-        sa.Column("host", sa.String(length=16), nullable=False),
-        sa.Column("url", sa.Text(), nullable=False),
-        sa.Column("label", sa.Text(), nullable=True),
-        sa.Column(
-            "status", sa.String(length=16), nullable=False,
-            server_default="pending",
-        ),
-        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("last_error", sa.Text(), nullable=True),
-        sa.Column(
-            "attachment_id", sa.Integer(),
-            sa.ForeignKey("post_attachment.id", ondelete="SET NULL"),
-            nullable=True,
-        ),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("duration_seconds", sa.Float(), nullable=True),
-        sa.CheckConstraint(
-            "host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')",
-            name="ck_external_link_host",
-        ),
-        sa.CheckConstraint(
-            "status IN ('pending','downloading','downloaded','failed',"
-            "'skipped','dead')",
-            name="ck_external_link_status",
-        ),
-    )
-    op.create_index(
-        "ix_external_link_post_id", "external_link", ["post_id"],
-    )
-    op.create_index(
-        "ix_external_link_artist_id", "external_link", ["artist_id"],
-    )
-    op.create_index(
-        "ix_external_link_status", "external_link", ["status"],
-    )
-    op.create_index(
-        "uq_external_link_post_url", "external_link", ["post_id", "url"],
-        unique=True,
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("uq_external_link_post_url", table_name="external_link")
-    op.drop_index("ix_external_link_status", table_name="external_link")
-    op.drop_index("ix_external_link_artist_id", table_name="external_link")
-    op.drop_index("ix_external_link_post_id", table_name="external_link")
-    op.drop_table("external_link")

alembic/versions/0050_external_link_host_toggles.py

@@ -1,38 +0,0 @@
-"""import_settings: per-host enable toggles for external file-host downloads
-
-Operator levers (#830): disable a single host (e.g. mega.nz when it's
-rate-limiting/banning) without touching the others. The worker reads these via
-getattr and defaults to enabled, so the toggles default TRUE (works out of the
-box, rule #26).
-
-Revision ID: 0050
-Revises: 0049
-Create Date: 2026-06-14
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0050"
-down_revision: Union[str, None] = "0049"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
-
-
-def upgrade() -> None:
-    for host in _HOSTS:
-        op.add_column(
-            "import_settings",
-            sa.Column(
-                f"extdl_{host}_enabled", sa.Boolean(), nullable=False,
-                server_default=sa.true(),
-            ),
-        )
-
-
-def downgrade() -> None:
-    for host in _HOSTS:
-        op.drop_column("import_settings", f"extdl_{host}_enabled")

alembic/versions/0051_image_source_provenance.py

@@ -1,38 +0,0 @@
-"""image_record: source_url + source_filehash (inline-image localization)
-
-#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline
-images instead of hotlinking the public CDN. The join key between a body
-`<img src=CDN>` and the local file is the CDN's 32-hex filehash (the same
-identity extract_media dedups by). Persist it (indexed) plus the full source
-URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing
-rows — those fall back to hotlinking until re-downloaded.
-
-Revision ID: 0051
-Revises: 0050
-Create Date: 2026-06-14
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0051"
-down_revision: Union[str, None] = "0050"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True))
-    op.add_column(
-        "image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True)
-    )
-    op.create_index(
-        "ix_image_record_source_filehash", "image_record", ["source_filehash"]
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_image_record_source_filehash", table_name="image_record")
-    op.drop_column("image_record", "source_filehash")
-    op.drop_column("image_record", "source_url")

alembic/versions/0052_image_duration_seconds.py

@@ -1,32 +0,0 @@
-"""image_record: duration_seconds (Tier-1 video near-dup key)
-
-#871. Videos previously deduped on sha256 only (pHash is images-only), so a
-different encode/remux of the same video imported as a distinct record. Persist
-the container duration so the importer can treat same-artist videos with matching
-duration (+ aspect ratio) as the same content and dedup/supersede like images.
-NULL for images and for video rows imported before this column existed (a
-backfill re-probes those so they participate in dedup).
-
-Revision ID: 0052
-Revises: 0051
-Create Date: 2026-06-16
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0052"
-down_revision: Union[str, None] = "0051"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "image_record", sa.Column("duration_seconds", sa.Float(), nullable=True)
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("image_record", "duration_seconds")

alembic/versions/0053_ml_settings_video_tagging.py

@@ -1,49 +0,0 @@
-"""ml_settings: video tagging knobs (cadence sampling + noise floor)
-
-#747. Video tag quality/perf: sample frames at a fixed cadence (interval) so a
-tag's frame-presence reflects real screen time, cap total frames so long videos
-stay bounded, and keep a tag only if it appears in >= min_tag_frames sampled
-frames. Operator-tunable via Settings → ML (replaces the VIDEO_ML_FRAMES env var).
-
-Revision ID: 0053
-Revises: 0052
-Create Date: 2026-06-16
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0053"
-down_revision: Union[str, None] = "0052"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "video_frame_interval_seconds", sa.Float(), nullable=False,
-            server_default="4.0",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "video_max_frames", sa.Integer(), nullable=False, server_default="64",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "video_min_tag_frames", sa.Integer(), nullable=False,
-            server_default="3",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "video_min_tag_frames")
-    op.drop_column("ml_settings", "video_max_frames")
-    op.drop_column("ml_settings", "video_frame_interval_seconds")

alembic/versions/0054_subscribestar_ledgers.py

@@ -1,82 +0,0 @@
-"""subscribestar_seen_media + subscribestar_failed_media: per-source ledgers
-
-Revision ID: 0054
-Revises: 0053
-Create Date: 2026-06-17
-
-SubscribeStar native ingester (phase 1 of the gallery-dl → native-core
-migration). Mirrors the Patreon ledger tables (0037/0038): a seen-ledger so
-routine walks skip already-ingested media (recovery bypasses it) and a
-dead-letter ledger so persistently-failing media stops re-burning backfill
-chunks. `filehash` is a CDN content hash when present, else a synthesized
-``<post_id>:<filename>`` key — hence String(128). UNIQUE (source_id, filehash)
-is the upsert key on each.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0054"
-down_revision: Union[str, None] = "0053"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "subscribestar_seen_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("post_id", sa.String(64), nullable=True),
-        sa.Column(
-            "seen_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_subscribestar_seen_media_source_id"
-        ),
-    )
-    op.create_table(
-        "subscribestar_failed_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
-        sa.Column("last_error", sa.Text, nullable=True),
-        sa.Column(
-            "first_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.Column(
-            "last_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_subscribestar_failed_media_source_id"
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("subscribestar_failed_media")
-    op.drop_table("subscribestar_seen_media")

alembic/versions/0055_image_provenance_from_attachment.py

@@ -1,55 +0,0 @@
-"""image_provenance: from_attachment_id (which archive an image was extracted from)
-
-Milestone #87. When an image is pulled out of a .zip/.rar, record WHICH archive
-PostAttachment it came from, so the provenance UI can show the single archive a
-file lives inside instead of every attachment on the post. Nullable FK with
-ON DELETE SET NULL — a loose (non-archive) download leaves it NULL, and deleting
-the archive attachment forgets the linkage without destroying the (image, post)
-provenance edge. Existing rows are NULL until the reextract backfill stamps them.
-
-Revision ID: 0055
-Revises: 0054
-Create Date: 2026-06-22
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0055"
-down_revision: Union[str, None] = "0054"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "image_provenance",
-        sa.Column("from_attachment_id", sa.Integer(), nullable=True),
-    )
-    op.create_index(
-        "ix_image_provenance_from_attachment_id",
-        "image_provenance",
-        ["from_attachment_id"],
-    )
-    op.create_foreign_key(
-        "fk_image_provenance_from_attachment",
-        "image_provenance",
-        "post_attachment",
-        ["from_attachment_id"],
-        ["id"],
-        ondelete="SET NULL",
-    )
-
-
-def downgrade() -> None:
-    op.drop_constraint(
-        "fk_image_provenance_from_attachment",
-        "image_provenance",
-        type_="foreignkey",
-    )
-    op.drop_index(
-        "ix_image_provenance_from_attachment_id",
-        table_name="image_provenance",
-    )
-    op.drop_column("image_provenance", "from_attachment_id")

alembic/versions/0056_tag_eval_run.py

@@ -1,43 +0,0 @@
-"""tag_eval_run: persisted head-vs-centroid tagging eval runs (#1130)
-
-Milestone #114 slice 1. A long ml-queue eval whose full report must SURVIVE
-navigation, so the run + report live in a row the admin card rehydrates from
-(mirrors library_audit_run). running -> ready / error.
-
-Revision ID: 0056
-Revises: 0055
-Create Date: 2026-06-28
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects.postgresql import JSONB
-
-revision: str = "0056"
-down_revision: Union[str, None] = "0055"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "tag_eval_run",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("params", JSONB(), nullable=False),
-        sa.Column("status", sa.String(length=16), nullable=False, server_default="running"),
-        sa.Column(
-            "started_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("report", JSONB(), nullable=True),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"])
-
-
-def downgrade() -> None:
-    op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run")
-    op.drop_table("tag_eval_run")

alembic/versions/0057_tag_positive_confirmation.py

@@ -1,40 +0,0 @@
-"""tag_positive_confirmation: operator-affirmed correct positives (#1130)
-
-Mirror of tag_suggestion_rejection. "Keep" on a doubted positive records here so
-the eval's doubts list stops resurfacing confirmed-correct images every run.
-
-Revision ID: 0057
-Revises: 0056
-Create Date: 2026-06-28
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0057"
-down_revision: Union[str, None] = "0056"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "tag_positive_confirmation",
-        sa.Column(
-            "image_record_id", sa.Integer(),
-            sa.ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True,
-        ),
-        sa.Column(
-            "confirmed_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("tag_positive_confirmation")

alembic/versions/0058_tag_head.py

@@ -1,95 +0,0 @@
-"""tag_head + head_training_run: production heads that learn from tags (#114)
-
-The eval (#1130) proved the frozen-embedding + trained-head spine; this lands its
-production form. tag_head stores one logistic-regression head per concept (the
-new suggestion source, replacing Camie + centroid); head_training_run tracks the
-batch that (re)trains them. Adds two head-training tunables to ml_settings.
-
-Revision ID: 0058
-Revises: 0057
-Create Date: 2026-06-28
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from pgvector.sqlalchemy import Vector
-from sqlalchemy.dialects.postgresql import JSONB
-
-revision: str = "0058"
-down_revision: Union[str, None] = "0057"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-_HEAD_DIM = 1152
-
-
-def upgrade() -> None:
-    op.create_table(
-        "tag_head",
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column("embedding_version", sa.String(length=128), nullable=False),
-        sa.Column("weights", Vector(_HEAD_DIM), nullable=False),
-        sa.Column("bias", sa.Float(), nullable=False),
-        sa.Column("suggest_threshold", sa.Float(), nullable=False),
-        sa.Column("auto_apply_threshold", sa.Float(), nullable=True),
-        sa.Column("n_pos", sa.Integer(), nullable=False),
-        sa.Column("n_neg", sa.Integer(), nullable=False),
-        sa.Column("ap", sa.Float(), nullable=False),
-        sa.Column("precision_cv", sa.Float(), nullable=False),
-        sa.Column("recall", sa.Float(), nullable=False),
-        sa.Column(
-            "trained_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("metrics", JSONB(), nullable=True),
-    )
-
-    op.create_table(
-        "head_training_run",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("params", JSONB(), nullable=False),
-        sa.Column(
-            "status", sa.String(length=16), nullable=False,
-            server_default="running",
-        ),
-        sa.Column(
-            "started_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("n_trained", sa.Integer(), nullable=True),
-        sa.Column("n_skipped", sa.Integer(), nullable=True),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.create_index(
-        "ix_head_training_run_status", "head_training_run", ["status"],
-    )
-
-    # Head-training tunables on the ml_settings singleton.
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "head_min_positives", sa.Integer(), nullable=False,
-            server_default="8",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "head_auto_apply_precision", sa.Float(), nullable=False,
-            server_default="0.97",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "head_auto_apply_precision")
-    op.drop_column("ml_settings", "head_min_positives")
-    op.drop_index("ix_head_training_run_status", table_name="head_training_run")
-    op.drop_table("head_training_run")
-    op.drop_table("tag_head")

alembic/versions/0059_head_auto_apply.py

@@ -1,70 +0,0 @@
-"""head_auto_apply_run + earned-auto-apply settings (#114)
-
-A graduated head can apply its tag without a human, gated by a master switch +
-a support floor. head_auto_apply_run tracks each sweep / dry-run preview.
-
-Revision ID: 0059
-Revises: 0058
-Create Date: 2026-06-29
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects.postgresql import JSONB
-
-revision: str = "0059"
-down_revision: Union[str, None] = "0058"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "head_auto_apply_run",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "dry_run", sa.Boolean(), nullable=False, server_default=sa.false()
-        ),
-        sa.Column("params", JSONB(), nullable=False),
-        sa.Column(
-            "status", sa.String(length=16), nullable=False,
-            server_default="running",
-        ),
-        sa.Column(
-            "started_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("n_applied", sa.Integer(), nullable=True),
-        sa.Column("report", JSONB(), nullable=True),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.create_index(
-        "ix_head_auto_apply_run_status", "head_auto_apply_run", ["status"],
-    )
-
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "head_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.true(),  # opt-out: on by default (operator-asked)
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "head_auto_apply_min_positives", sa.Integer(), nullable=False,
-            server_default="30",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "head_auto_apply_min_positives")
-    op.drop_column("ml_settings", "head_auto_apply_enabled")
-    op.drop_index(
-        "ix_head_auto_apply_run_status", table_name="head_auto_apply_run"
-    )
-    op.drop_table("head_auto_apply_run")

alembic/versions/0060_head_metrics.py

@@ -1,74 +0,0 @@
-"""head_metric + head_metrics_snapshot: auto-apply observability (#114)
-
-Running misfire/under-fire counters per concept (captured at correction time,
-since image_tag.source is lost on delete) + a daily per-concept time-series so
-the operator can tune the precision target + support floor from real data.
-
-Revision ID: 0060
-Revises: 0059
-Create Date: 2026-06-29
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0060"
-down_revision: Union[str, None] = "0059"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "head_metric",
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column(
-            "updated_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-
-    op.create_table(
-        "head_metrics_snapshot",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"),
-        ),
-        sa.Column("name", sa.String(length=255), nullable=False),
-        sa.Column(
-            "snapshot_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("n_auto_applied", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("ap", sa.Float(), nullable=True),
-        sa.Column("precision_cv", sa.Float(), nullable=True),
-        sa.Column("recall", sa.Float(), nullable=True),
-        sa.Column("n_pos", sa.Integer(), nullable=True),
-    )
-    op.create_index(
-        "ix_head_metrics_snapshot_tag_id", "head_metrics_snapshot", ["tag_id"],
-    )
-    op.create_index(
-        "ix_head_metrics_snapshot_snapshot_at", "head_metrics_snapshot",
-        ["snapshot_at"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_head_metrics_snapshot_snapshot_at", table_name="head_metrics_snapshot"
-    )
-    op.drop_index(
-        "ix_head_metrics_snapshot_tag_id", table_name="head_metrics_snapshot"
-    )
-    op.drop_table("head_metrics_snapshot")
-    op.drop_table("head_metric")

alembic/versions/0061_image_region.py

@@ -1,59 +0,0 @@
-"""image_region: detected/proposed regions + their crop embeddings (#114)
-
-Storage backbone of the crop pipeline. A region = normalized bbox + the crop's
-embedding (CCIP for face/figure → character id; SigLIP for concept regions →
-head bag-of-embeddings). Also serves as grounded-tag bbox provenance.
-
-Revision ID: 0061
-Revises: 0060
-Create Date: 2026-06-29
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from pgvector.sqlalchemy import Vector
-
-revision: str = "0061"
-down_revision: Union[str, None] = "0060"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-_CCIP_DIM = 768
-_SIGLIP_DIM = 1152
-
-
-def upgrade() -> None:
-    op.create_table(
-        "image_region",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "image_record_id", sa.Integer(),
-            sa.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False,
-        ),
-        sa.Column("kind", sa.String(length=16), nullable=False),
-        # Video/animated: source frame timestamp (seconds); NULL for stills.
-        sa.Column("frame_time", sa.Float(), nullable=True),
-        sa.Column("rx", sa.Float(), nullable=False),
-        sa.Column("ry", sa.Float(), nullable=False),
-        sa.Column("rw", sa.Float(), nullable=False),
-        sa.Column("rh", sa.Float(), nullable=False),
-        sa.Column("score", sa.Float(), nullable=True),
-        sa.Column("detector_version", sa.String(length=64), nullable=True),
-        sa.Column("crop_version", sa.String(length=64), nullable=True),
-        sa.Column("embedding_version", sa.String(length=128), nullable=True),
-        sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=True),
-        sa.Column("siglip_embedding", Vector(_SIGLIP_DIM), nullable=True),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-    op.create_index(
-        "ix_image_region_image_record_id", "image_region", ["image_record_id"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_image_region_image_record_id", table_name="image_region")
-    op.drop_table("image_region")

alembic/versions/0062_gpu_job.py

@@ -1,55 +0,0 @@
-"""gpu_job: the HTTP-leased GPU work queue for the desktop agent (#114)
-
-The agent stays HTTP-only — the server enqueues per-(image, task) jobs here and
-the agent leases/submits over the web API; Redis/Postgres stay private.
-
-Revision ID: 0062
-Revises: 0061
-Create Date: 2026-06-29
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0062"
-down_revision: Union[str, None] = "0061"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "gpu_job",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "image_record_id", sa.Integer(),
-            sa.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False,
-        ),
-        sa.Column("task", sa.String(length=32), nullable=False),
-        sa.Column(
-            "status", sa.String(length=16), nullable=False,
-            server_default="pending",
-        ),
-        sa.Column("lease_token", sa.String(length=64), nullable=True),
-        sa.Column("leased_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column(
-            "updated_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-    op.create_index("ix_gpu_job_image_record_id", "gpu_job", ["image_record_id"])
-    op.create_index("ix_gpu_job_status", "gpu_job", ["status"])
-
-
-def downgrade() -> None:
-    op.drop_index("ix_gpu_job_status", table_name="gpu_job")
-    op.drop_index("ix_gpu_job_image_record_id", table_name="gpu_job")
-    op.drop_table("gpu_job")

alembic/versions/0063_ccip_match_threshold.py

@@ -1,33 +0,0 @@
-"""ml_settings.ccip_match_threshold — tunable CCIP character-match cut (#114)
-
-The v1 matcher used a flat 0.75 cosine; live data showed that over-fires (a
-high-reference character matched a scatter of images). 0.85 keeps the confident
-single-character matches and drops the noise. Tunable from the GPU agent card.
-
-Revision ID: 0063
-Revises: 0062
-Create Date: 2026-06-29
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0063"
-down_revision: Union[str, None] = "0062"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_match_threshold", sa.Float(), nullable=False,
-            server_default="0.85",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "ccip_match_threshold")

alembic/versions/0064_ccip_auto_apply.py

@@ -1,42 +0,0 @@
-"""ml_settings: CCIP auto-apply switch + threshold (#114)
-
-Confident CCIP character matches auto-tag (source='ccip_auto') on a daily sweep,
-so identity tags keep flowing without pressing a button. ON by default (opt-out,
-like head auto-apply); the high threshold (0.92, above the 0.85 suggest cut) +
-single-character references keep it safe, and every auto-tag is reversible.
-
-Revision ID: 0064
-Revises: 0063
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0064"
-down_revision: Union[str, None] = "0063"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.true(),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_auto_apply_threshold", sa.Float(), nullable=False,
-            server_default="0.92",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "ccip_auto_apply_threshold")
-    op.drop_column("ml_settings", "ccip_auto_apply_enabled")

alembic/versions/0065_embedder_model_name.py

@@ -1,35 +0,0 @@
-"""ml_settings: embedder_model_name (#1190 operator model swap)
-
-The embedder MODEL VERSION was already a setting (and stamps image_record.
-siglip_model_version); the HF model NAME was env-only, so an operator couldn't
-actually point the pipeline at a different embedder. Storing the name as a
-setting makes the model an operator choice: set name + version → re-embed (the
-GPU agent) → retrain heads. Default = the current SigLIP so400m.
-
-Revision ID: 0065
-Revises: 0064
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0065"
-down_revision: Union[str, None] = "0064"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "embedder_model_name", sa.String(length=128), nullable=False,
-            server_default="google/siglip-so400m-patch14-384",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "embedder_model_name")

alembic/versions/0066_drop_centroids.py

@@ -1,57 +0,0 @@
-"""drop the dead per-tag centroid subsystem (#1189 cleanup)
-
-The v2 pivot replaced per-tag SigLIP centroids with learned heads + CCIP.
-Nothing read the centroids anymore — they were recomputed (on merge + a daily
-beat) but never consumed for suggestions or auto-apply. Remove the storage +
-its two now-unused settings columns. (The recompute tasks, beat, endpoint,
-service, and UI card are removed in the same change.)
-
-Revision ID: 0066
-Revises: 0065
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0066"
-down_revision: Union[str, None] = "0065"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_table("tag_reference_embedding")
-    op.drop_column("ml_settings", "centroid_similarity_threshold")
-    op.drop_column("ml_settings", "min_reference_images")
-
-
-def downgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "min_reference_images", sa.Integer(), nullable=False,
-            server_default="5",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "centroid_similarity_threshold", sa.Float(), nullable=False,
-            server_default="0.55",
-        ),
-    )
-    op.create_table(
-        "tag_reference_embedding",
-        sa.Column("tag_id", sa.Integer(), nullable=False),
-        sa.Column("embedding", sa.LargeBinary(), nullable=False),
-        sa.Column("reference_count", sa.Integer(), nullable=False),
-        sa.Column("model_version", sa.String(length=128), nullable=False),
-        sa.Column(
-            "updated_at", sa.DateTime(timezone=True),
-            server_default=sa.func.now(), nullable=False,
-        ),
-        sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("tag_id"),
-    )

alembic/versions/0067_retire_camie_allowlist.py

@@ -1,66 +0,0 @@
-"""retire the Camie tagger + allowlist bulk-apply (#1189)
-
-The v2 pivot made heads + CCIP the tag source and head auto-apply the earned
-propagation. The Camie tagger ran only to feed the allowlist bulk-apply (its
-predictions had no other consumer), and the allowlist was a second, un-earned
-auto-apply path parallel to heads. Both are retired — drop their storage.
-
-(image_prediction = Camie's per-image predictions; tag_allowlist = the bulk-
-apply allowlist. Nothing references INTO these tables, so the drop is clean.)
-
-Revision ID: 0067
-Revises: 0066
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0067"
-down_revision: Union[str, None] = "0066"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_table("image_prediction")
-    op.drop_table("tag_allowlist")
-
-
-def downgrade() -> None:
-    op.create_table(
-        "tag_allowlist",
-        sa.Column("tag_id", sa.Integer(), nullable=False),
-        sa.Column(
-            "min_confidence", sa.Float(), nullable=False, server_default="0.9"
-        ),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True),
-            server_default=sa.func.now(), nullable=False,
-        ),
-        sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("tag_id"),
-        sa.CheckConstraint(
-            "min_confidence >= 0 AND min_confidence <= 1",
-            name="ck_tag_allowlist_confidence_range",
-        ),
-    )
-    op.create_table(
-        "image_prediction",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("image_record_id", sa.Integer(), nullable=False),
-        sa.Column("raw_name", sa.String(length=255), nullable=False),
-        sa.Column("category", sa.String(length=32), nullable=False),
-        sa.Column("score", sa.Float(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["image_record_id"], ["image_record.id"], ondelete="CASCADE"
-        ),
-    )
-    op.create_index(
-        "ix_image_prediction_image", "image_prediction", ["image_record_id"]
-    )
-    op.create_index(
-        "ix_image_prediction_name_score", "image_prediction",
-        ["raw_name", "score"],
-    )

alembic/versions/0068_drop_dead_tagger_settings.py

@@ -1,80 +0,0 @@
-"""drop dead tagger/suggestion settings + columns left after Camie retirement (#1199)
-
-Hygiene follow-up to #1189. These were left inert to bound that change; nothing
-reads them now:
-- ml_settings: tagger_store_floor + tagger_model_version (only the deleted Camie
-  tagger used them), suggestion_threshold_character/general (already dead pre-
-  retirement — scoring uses per-head thresholds), video_min_tag_frames (only the
-  deleted video-prediction aggregator used it).
-- image_record: tagger_model_version (no writer now), centroid_scores (long-dead
-  JSON cache, no reader).
-
-Revision ID: 0068
-Revises: 0067
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0068"
-down_revision: Union[str, None] = "0067"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_column("ml_settings", "suggestion_threshold_character")
-    op.drop_column("ml_settings", "suggestion_threshold_general")
-    op.drop_column("ml_settings", "tagger_store_floor")
-    op.drop_column("ml_settings", "video_min_tag_frames")
-    op.drop_column("ml_settings", "tagger_model_version")
-    op.drop_column("image_record", "tagger_model_version")
-    op.drop_column("image_record", "centroid_scores")
-
-
-def downgrade() -> None:
-    op.add_column(
-        "image_record",
-        sa.Column("centroid_scores", sa.JSON(), nullable=True),
-    )
-    op.add_column(
-        "image_record",
-        sa.Column("tagger_model_version", sa.String(length=128), nullable=True),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "tagger_model_version", sa.String(length=128), nullable=False,
-            server_default="camie-tagger-v2",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "video_min_tag_frames", sa.Integer(), nullable=False,
-            server_default="3",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "tagger_store_floor", sa.Float(), nullable=False,
-            server_default="0.7",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "suggestion_threshold_general", sa.Float(), nullable=False,
-            server_default="0.7",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "suggestion_threshold_character", sa.Float(), nullable=False,
-            server_default="0.7",
-        ),
-    )

alembic/versions/0069_default_siglip2.py

@@ -1,51 +0,0 @@
-"""default the embedder to SigLIP 2 — for FRESH installs only (#1203)
-
-Make SigLIP 2 (so400m, 512px; a 1152-d drop-in) the default embedder. New
-installs start on it. An EXISTING library is NOT touched: flipping its stored
-embedder version would mark every embedding stale (the scorer is version-gated)
-and kill suggestions until a full re-embed+retrain — so an existing instance
-switches deliberately via Settings → GPU agent → Embedding model → Re-embed →
-Retrain. We detect "fresh" by the absence of any embedded image.
-
-Revision ID: 0069
-Revises: 0068
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0069"
-down_revision: Union[str, None] = "0068"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-_NEW_NAME = "google/siglip2-so400m-patch16-512"
-_NEW_VERSION = "siglip2-so400m-patch16-512"
-_OLD_NAME = "google/siglip-so400m-patch14-384"
-_OLD_VERSION = "siglip-so400m-patch14-384"
-
-
-def upgrade() -> None:
-    # Fresh install (nothing embedded yet) → adopt SigLIP 2.
-    op.execute(
-        f"""
-        UPDATE ml_settings SET
-            embedder_model_name = '{_NEW_NAME}',
-            embedder_model_version = '{_NEW_VERSION}'
-        WHERE NOT EXISTS (
-            SELECT 1 FROM image_record WHERE siglip_embedding IS NOT NULL
-        )
-        """
-    )
-    op.alter_column("ml_settings", "embedder_model_name", server_default=_NEW_NAME)
-    op.alter_column(
-        "ml_settings", "embedder_model_version", server_default=_NEW_VERSION
-    )
-
-
-def downgrade() -> None:
-    op.alter_column("ml_settings", "embedder_model_name", server_default=_OLD_NAME)
-    op.alter_column(
-        "ml_settings", "embedder_model_version", server_default=_OLD_VERSION
-    )

alembic/versions/0070_gpu_job_lease_indexes.py

@@ -1,44 +0,0 @@
-"""partial indexes so GPU-job leasing stays O(batch), not O(completed)
-
-The lease claims the lowest-id pending (or expired-leased) jobs. With only a
-plain `status` index, `... ORDER BY id LIMIT n` walked the primary-key index from
-the start, skipping the entire prefix of already-done/error rows before reaching
-pending ones — so leasing slowed to a crawl as `done` piled up (the whole reason
-throughput fell off a cliff mid-run and /status stalled). Two partial indexes fix
-it: the pending one is id-ordered so the hot path reads just the first n entries,
-and the leased-expiry one keeps the crash-recovery reclaim + the orphan sweep
-cheap. They cover only the small live slice of the table, so they stay tiny even
-as the done/error history grows to millions.
-
-Revision ID: 0070
-Revises: 0069
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0070"
-down_revision: Union[str, None] = "0069"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Hot path: lowest-id pending jobs. Index on id, restricted to pending, so
-    # `WHERE status='pending' ORDER BY id LIMIT n` is a short index-order scan.
-    op.create_index(
-        "ix_gpu_job_pending", "gpu_job", ["id"],
-        postgresql_where=sa.text("status = 'pending'"),
-    )
-    # Crash-recovery: expired leases, for the lease backstop + recover_orphaned.
-    op.create_index(
-        "ix_gpu_job_leased_expires", "gpu_job", ["lease_expires_at"],
-        postgresql_where=sa.text("status = 'leased'"),
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_gpu_job_leased_expires", table_name="gpu_job")
-    op.drop_index("ix_gpu_job_pending", table_name="gpu_job")

alembic/versions/0071_image_record_earliest_post_date.py

@@ -1,80 +0,0 @@
-"""image_record.earliest_post_date: original-publish gallery sort key + index
-
-Revision ID: 0071
-Revises: 0070
-Create Date: 2026-07-01
-
-effective_date (0035) keys off the PRIMARY post — which is often the repost /
-download the file actually came from — and falls back to created_at, so the
-gallery's default order surfaces download dates rather than when content was
-first posted (operator-flagged 2026-07-01). Materialize a second sort key,
-earliest_post_date = MIN(post_date) across ALL of an image's provenance posts
-(every post it appears in), falling back to created_at only when no linked post
-carries a date. Indexed (DESC, id DESC) so the "post date" gallery sort is an
-index range scan just like effective_date.
-
-Backfill mirrors 0035: created_at baseline, then override with the MIN over
-image_provenance ⋈ post. New rows get the created_at-equivalent server default;
-services/importer.py recomputes it whenever a dated post is linked.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0071"
-down_revision: Union[str, None] = "0070"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Add nullable first so the backfill can populate before NOT NULL.
-    op.add_column(
-        "image_record",
-        sa.Column("earliest_post_date", sa.DateTime(timezone=True), nullable=True),
-    )
-    # Baseline: download date. Set-based (no per-row binds) → immune to the
-    # 65535 bind-parameter ceiling regardless of library size.
-    op.execute(
-        """
-        UPDATE image_record
-        SET earliest_post_date = created_at
-        """
-    )
-    # Override with the earliest post_date across EVERY post the image appears
-    # in (image_provenance is the many-to-many edge; ignore posts with no date).
-    op.execute(
-        """
-        UPDATE image_record AS ir
-        SET earliest_post_date = sub.min_date
-        FROM (
-            SELECT ip.image_record_id AS iid, MIN(p.post_date) AS min_date
-            FROM image_provenance AS ip
-            JOIN post AS p ON p.id = ip.post_id
-            WHERE p.post_date IS NOT NULL
-            GROUP BY ip.image_record_id
-        ) AS sub
-        WHERE ir.id = sub.iid
-        """
-    )
-    op.alter_column(
-        "image_record",
-        "earliest_post_date",
-        nullable=False,
-        server_default=sa.text("now()"),
-    )
-    # DESC/DESC matches the gallery's ORDER BY earliest_post_date DESC, id DESC
-    # so the "post date" scroll is a forward index scan; raw SQL because
-    # alembic's column list doesn't express per-column DESC cleanly.
-    op.execute(
-        "CREATE INDEX ix_image_record_earliest_post_date "
-        "ON image_record (earliest_post_date DESC, id DESC)"
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_image_record_earliest_post_date", table_name="image_record"
-    )
-    op.drop_column("image_record", "earliest_post_date")

alembic/versions/0072_gpu_job_triage_status.py

@@ -1,32 +0,0 @@
-"""gpu_job.triage_status — the probe's verdict on an errored job's FILE
-
-Failure triage (#125): a periodic sweep probes each errored image's file
-(sha256 + decode, verify_integrity's machinery) exactly once and stores the
-verdict here — 'defect' (the file is bad: recovery material, excluded from
-/retry_errors) or 'file_ok' (failure was operational, safe to retry). NULL
-means not yet probed; selecting on NULL is what makes the sweep resumable.
-No index: the errored slice the sweep scans is tiny by design (tombstones).
-
-Revision ID: 0072
-Revises: 0071
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0072"
-down_revision: Union[str, None] = "0071"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "gpu_job", sa.Column("triage_status", sa.String(16), nullable=True)
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("gpu_job", "triage_status")

alembic/versions/0073_drop_tag_eval_run.py

@@ -1,46 +0,0 @@
-"""drop tag_eval_run — the head-vs-centroid eval harness is retired
-
-The eval (#1130) existed to prove the heads tagging spine on the operator's own
-data. It did; the operator accepted the system and retired the harness
-(2026-07-02) — card, API, task, model and this table all go. The eval's data
-loaders + metric helpers live on in services/ml/training_data.py, where the
-production heads trainer uses them nightly.
-
-Revision ID: 0073
-Revises: 0072
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-revision: str = "0073"
-down_revision: Union[str, None] = "0072"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run")
-    op.drop_table("tag_eval_run")
-
-
-def downgrade() -> None:
-    # Recreates the shape from 0056 (data is not restorable).
-    op.create_table(
-        "tag_eval_run",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("params", postgresql.JSONB(), nullable=False),
-        sa.Column("status", sa.String(length=16), nullable=False,
-                  server_default="running"),
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("report", postgresql.JSONB(), nullable=True),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column("last_progress_at", sa.DateTime(timezone=True),
-                  nullable=True),
-    )
-    op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"])

alembic/versions/0074_ml_settings_cpu_embed_enabled.py

@@ -1,35 +0,0 @@
-"""ml_settings.cpu_embed_enabled — the CPU embed fallback becomes a switch
-
-B3 (operator 2026-07-02): the ml-worker's only processing role is the CPU
-whole-image embed for stacks without a GPU agent. ON by default (a fresh
-install works agent-less); agent-equipped stacks that drop the ml-worker
-container turn it off so import hooks stop queueing embed work into a queue
-nothing consumes — the daily GPU 'embed' backfill covers those images.
-
-Revision ID: 0074
-Revises: 0073
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0074"
-down_revision: Union[str, None] = "0073"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "cpu_embed_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.true(),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "cpu_embed_enabled")

alembic/versions/0075_tag_is_system.py

@@ -1,60 +0,0 @@
-"""tag.is_system + seed the three hygiene system tags
-
-Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a
-character poison that character's head and CCIP references; banners/editor
-screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the
-product ships — not operator configuration — so the seed lives here.
-
-Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive,
-matching TagService.rename's collision stance) instead of inserting a
-duplicate, so an operator who already tagged `wip` keeps their applications.
-
-Revision ID: 0075
-Revises: 0074
-Create Date: 2026-07-03
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0075"
-down_revision: Union[str, None] = "0074"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
-
-
-def upgrade() -> None:
-    op.add_column(
-        "tag",
-        sa.Column(
-            "is_system", sa.Boolean(), nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-    conn = op.get_bind()
-    for name in SYSTEM_TAG_NAMES:
-        adopted = conn.execute(
-            sa.text(
-                "UPDATE tag SET is_system = true "
-                "WHERE lower(name) = lower(:name) AND kind = 'general'"
-            ),
-            {"name": name},
-        )
-        if adopted.rowcount == 0:
-            conn.execute(
-                sa.text(
-                    "INSERT INTO tag (name, kind, is_system) "
-                    "VALUES (:name, 'general', true)"
-                ),
-                {"name": name},
-            )
-
-
-def downgrade() -> None:
-    # The seeded rows survive as ordinary general tags — dropping the flag is
-    # enough to disarm the mechanism, and deleting rows would orphan any
-    # operator applications made while the flag existed.
-    op.drop_column("tag", "is_system")

alembic/versions/0076_pixiv_ledgers.py

@@ -1,82 +0,0 @@
-"""pixiv_seen_media + pixiv_failed_media: per-source ledgers
-
-Revision ID: 0076
-Revises: 0075
-Create Date: 2026-07-03
-
-Pixiv native ingester (milestone #129, gallery-dl → native-core migration).
-Mirrors the Patreon (0037/0038) and SubscribeStar (0054) ledger tables: a
-seen-ledger so routine walks skip already-ingested media (recovery bypasses
-it) and a dead-letter ledger so persistently-failing media stops re-burning
-backfill chunks. Pixiv URLs carry no content hash, so `filehash` is always the
-synthesized ``<illust_id>:p<num>`` / ``<illust_id>:ugoira`` key — String(128)
-matches the siblings. UNIQUE (source_id, filehash) is the upsert key on each.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0076"
-down_revision: Union[str, None] = "0075"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "pixiv_seen_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("post_id", sa.String(64), nullable=True),
-        sa.Column(
-            "seen_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_seen_media_source_id"
-        ),
-    )
-    op.create_table(
-        "pixiv_failed_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
-        sa.Column("last_error", sa.Text, nullable=True),
-        sa.Column(
-            "first_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.Column(
-            "last_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_failed_media_source_id"
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("pixiv_failed_media")
-    op.drop_table("pixiv_seen_media")

alembic/versions/0077_artist_name_not_unique.py

@@ -1,32 +0,0 @@
-"""drop uq_artist_name — decouple display name from identity/storage
-
-Revision ID: 0077
-Revises: 0076
-Create Date: 2026-07-04
-
-Artist model fragility fix (milestone #130). One `slug` column was doing
-identity + storage-path + display, and BOTH `name` and `slug` were UNIQUE, so
-the display name couldn't be edited freely and two genuinely different creators
-collided. Decouple: `slug` stays the immutable, unique storage/identity key (the
-on-disk path component — untouched here); `name` becomes freely editable, NON-
-unique display text. This migration only drops the `uq_artist_name` constraint;
-no data moves and no path changes.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0077"
-down_revision: Union[str, None] = "0076"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_constraint("uq_artist_name", "artist", type_="unique")
-
-
-def downgrade() -> None:
-    # Re-adding the UNIQUE would fail if duplicate names now exist; callers that
-    # need to reverse this must dedupe names first.
-    op.create_unique_constraint("uq_artist_name", "artist", ["name"])

alembic/versions/0078_ml_settings_detectors.py

@@ -1,83 +0,0 @@
-"""ml_settings crop-proposer / detector config (#134)
-
-Move the WHERE-to-crop detector config (per-proposer enable + weights + conf,
-plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU
-agent in the lease (like the embedder model) — no restart, agent env is now
-bootstrap-only. All server_defaults are the working values so existing rows +
-fresh installs crop out-of-the-box with all three proposers ON.
-
-Revision ID: 0078
-Revises: 0077
-Create Date: 2026-07-05
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0078"
-down_revision: Union[str, None] = "0077"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-_ANATOMY_DEFAULT = (
-    "https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt"
-)
-_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt"
-
-
-def upgrade() -> None:
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_weights", sa.String(512), nullable=False,
-        server_default="yolo11n.pt"))
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.35")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_weights", sa.String(512), nullable=False,
-        server_default=_ANATOMY_DEFAULT))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.30")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_weights", sa.String(512), nullable=False,
-        server_default=_PANEL_DEFAULT))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.30")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_figures", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_components", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_panels", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_regions", sa.Integer(), nullable=False,
-        server_default=sa.text("128")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_dedupe_iou", sa.Float(), nullable=False,
-        server_default=sa.text("0.85")))
-
-
-def downgrade() -> None:
-    for col in (
-        "detector_person_enabled", "detector_person_weights", "detector_person_conf",
-        "detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf",
-        "detector_panel_enabled", "detector_panel_weights", "detector_panel_conf",
-        "detector_max_figures", "detector_max_components", "detector_max_panels",
-        "detector_max_regions", "detector_dedupe_iou",
-    ):
-        op.drop_column("ml_settings", col)

alembic/versions/0079_character_prototypes.py

@@ -1,77 +0,0 @@
-"""character prototype store (#1317) — precomputed, incremental CCIP references
-
-New tables character_prototype + ccip_prototype_state, plus MLSettings columns
-ccip_ref_signature (cheap global change gate) + ccip_prototype_cap (per-character
-reference cap). The reference set the CCIP matcher uses becomes a precomputed
-artifact refreshed incrementally off the request path. See milestone 138 /
-backend.app.services.ml.character_prototypes.
-
-Revision ID: 0079
-Revises: 0078
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from pgvector.sqlalchemy import Vector
-
-revision: str = "0079"
-down_revision: Union[str, None] = "0078"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-# Matches models.image_region.CCIP_DIM (the CCIP figure-embedding width).
-_CCIP_DIM = 768
-
-
-def upgrade() -> None:
-    op.create_table(
-        "character_prototype",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), nullable=False,
-        ),
-        sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=False),
-        sa.Column(
-            "region_id", sa.Integer(),
-            sa.ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True,
-        ),
-    )
-    op.create_index(
-        "ix_character_prototype_tag_id", "character_prototype", ["tag_id"]
-    )
-    op.create_table(
-        "ccip_prototype_state",
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column("fingerprint", sa.String(64), nullable=False),
-        sa.Column(
-            "updated_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column("ccip_ref_signature", sa.String(128), nullable=True),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_prototype_cap", sa.Integer(), nullable=False,
-            server_default=sa.text("64"),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "ccip_prototype_cap")
-    op.drop_column("ml_settings", "ccip_ref_signature")
-    op.drop_table("ccip_prototype_state")
-    op.drop_index(
-        "ix_character_prototype_tag_id", table_name="character_prototype"
-    )
-    op.drop_table("character_prototype")

alembic/versions/0080_tag_head_train_fingerprint.py

@@ -1,31 +0,0 @@
-"""tag_head.train_fingerprint (#1317 phase 2) — incremental head retraining
-
-A per-head training-data fingerprint (positive + rejection count/latest-timestamp)
-so a manual Retrain refits only the tags whose data changed; the nightly run
-ignores it (full reconcile). Nullable — a NULL fingerprint (existing heads) forces
-a refit on the first incremental run, then it's stamped.
-
-Revision ID: 0080
-Revises: 0079
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0080"
-down_revision: Union[str, None] = "0079"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "tag_head",
-        sa.Column("train_fingerprint", sa.String(128), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("tag_head", "train_fingerprint")

alembic/versions/0081_stricter_auto_apply_defaults.py

@@ -1,43 +0,0 @@
-"""stricter auto-apply defaults (milestone 139) — cut auto-apply misfires
-
-head_auto_apply_min_positives 30→50 and ccip_auto_apply_threshold 0.92→0.95
-(operator-asked 2026-07-06). The head graduation precision bar stays 0.97 — the
-operator confirmed the general-tag confidence was already well tuned; only the
-support floor + the CCIP match confidence are raised. The model defaults change
-for fresh installs; here we bump the existing singleton row IFF it is still at
-the previous default, so a deliberate operator change is NOT clobbered.
-
-Revision ID: 0081
-Revises: 0080
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0081"
-down_revision: Union[str, None] = "0080"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings SET head_auto_apply_min_positives = 50 "
-        "WHERE head_auto_apply_min_positives = 30"
-    )
-    op.execute(
-        "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.95 "
-        "WHERE ccip_auto_apply_threshold = 0.92"
-    )
-
-
-def downgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings SET head_auto_apply_min_positives = 30 "
-        "WHERE head_auto_apply_min_positives = 50"
-    )
-    op.execute(
-        "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.92 "
-        "WHERE ccip_auto_apply_threshold = 0.95"
-    )

alembic/versions/0082_presentation_auto_hide.py

@@ -1,85 +0,0 @@
-"""presentation-chrome auto-hide (#141) — settings knobs + review table
-
-MLSettings gains presentation_auto_apply_enabled / _threshold and
-presentation_conflict_threshold: banner + editor-screenshot auto-hide on the
-sweep with a FLAT threshold (decoupled from content-head graduation), and a
-conflict threshold that flags an auto-hide that "also looks like content".
-
-New table presentation_review records an auto-hidden chrome image that also
-scored high on a content head, surfaced in the Hidden view for a keep-hidden /
-un-hide decision. Resolved rows are pruned by retention.
-
-Revision ID: 0082
-Revises: 0081
-Create Date: 2026-07-07
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0082"
-down_revision: Union[str, None] = "0081"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("true"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_auto_apply_threshold", sa.Float(), nullable=False,
-            server_default=sa.text("0.90"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_conflict_threshold", sa.Float(), nullable=False,
-            server_default=sa.text("0.50"),
-        ),
-    )
-    op.create_table(
-        "presentation_review",
-        sa.Column(
-            "image_record_id", sa.Integer(),
-            sa.ForeignKey("image_record.id", ondelete="CASCADE"),
-            primary_key=True,
-        ),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column(
-            "conflict_tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True,
-        ),
-        sa.Column("conflict_score", sa.Float(), nullable=False),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    # The review list queries the unresolved flags (resolved_at IS NULL).
-    op.create_index(
-        "ix_presentation_review_resolved_at", "presentation_review",
-        ["resolved_at"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_presentation_review_resolved_at", table_name="presentation_review"
-    )
-    op.drop_table("presentation_review")
-    op.drop_column("ml_settings", "presentation_conflict_threshold")
-    op.drop_column("ml_settings", "presentation_auto_apply_threshold")
-    op.drop_column("ml_settings", "presentation_auto_apply_enabled")

alembic/versions/0083_post_translation.py

@@ -1,73 +0,0 @@
-"""post-text translation via Interpreter (milestone 143) — Post columns + settings
-
-Post gains the translated title/description + the detected source language,
-Interpreter engine_version (cache key), and translated_at — filled by the
-translate sweep. ImportSettings gains translation_enabled (OFF by default),
-interpreter_base_url (EMPTY — the operator sets their own, behind a reverse
-proxy), and translation_target_lang (en).
-
-Revision ID: 0083
-Revises: 0082
-Create Date: 2026-07-07
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0083"
-down_revision: Union[str, None] = "0082"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "post", sa.Column("post_title_translated", sa.Text(), nullable=True)
-    )
-    op.add_column(
-        "post", sa.Column("description_translated", sa.Text(), nullable=True)
-    )
-    op.add_column(
-        "post",
-        sa.Column("translated_source_lang", sa.String(8), nullable=True),
-    )
-    op.add_column(
-        "post",
-        sa.Column("translation_engine_version", sa.String(128), nullable=True),
-    )
-    op.add_column(
-        "post",
-        sa.Column("translated_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("false"),
-        ),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "interpreter_base_url", sa.Text(), nullable=False, server_default="",
-        ),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_target_lang", sa.Text(), nullable=False,
-            server_default="en",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "translation_target_lang")
-    op.drop_column("import_settings", "interpreter_base_url")
-    op.drop_column("import_settings", "translation_enabled")
-    op.drop_column("post", "translated_at")
-    op.drop_column("post", "translation_engine_version")
-    op.drop_column("post", "translated_source_lang")
-    op.drop_column("post", "description_translated")
-    op.drop_column("post", "post_title_translated")

backend/app/__init__.py

@@ -3,23 +3,13 @@
 import logging
 from pathlib import Path
 
-from quart import Quart, request
+from quart import Quart
 
 from .api import all_blueprints
 from .config import get_config
 from .frontend import frontend_bp
 from .services.credential_crypto import CredentialCrypto
 
-# Browser-extension origins. The FabledCurator extension fetches from
-# moz-extension://<uuid>/ on Firefox and chrome-extension://<uuid>/ on
-# Chromium-based browsers. Operator-flagged 2026-05-26: extension's
-# 'Test connection' returned `NetworkError` because the X-Extension-Key
-# header on /api/credentials triggers a CORS preflight that our routes
-# don't handle. Whitelisting only these two schemes (not opening CORS
-# up generally) lets the extension talk to a plain-HTTP self-hosted FC
-# without weakening the no-CORS posture for normal browser usage.
-_EXTENSION_ORIGIN_SCHEMES = ("moz-extension://", "chrome-extension://")
-
 _CREDENTIAL_KEY_PATH = Path("/images/secrets/credential_key.b64")
 
 
@@ -33,56 +23,18 @@ def create_app() -> Quart:
 
     app = Quart(__name__)
     app.secret_key = cfg.secret_key
-
-    # Stream files in 4 MiB chunks instead of Quart's 8 KiB default. The image
-    # library lives on a CIFS/SMB share (mounted rsize=4 MiB), so 8 KiB reads
-    # meant ~19k network round-trips for one large original — 30–58s downloads
-    # that starved both the GPU agent and the browser (operator-flagged
-    # 2026-07-01). 4 MiB matches the mount's read size → one round-trip per read,
-    # ~500× fewer. buffer_size is the MAX read, so small thumbnails still read in
-    # a single gulp, and Range/mime/ETag/conditional handling lives on Response,
-    # so this keeps all of it. Guarded so a future Quart-internal change can't
-    # break boot — worst case we fall back to the slow default.
-    try:
-        from quart.wrappers.response import FileBody
-        FileBody.buffer_size = 4 * 1024 * 1024
-    except Exception:
-        logging.getLogger(__name__).warning(
-            "could not raise FileBody.buffer_size — file serving stays on 8 KiB chunks"
-        )
+    # FC-5: legacy IR ingest JSON can run to tens of MB (hundreds of
+    # thousands of image_tag_associations). Werkzeug's default form
+    # memory cap is 500KB; raise both ceilings so the multipart upload
+    # for /api/migrate/ir_ingest doesn't 413.
+    app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024   # 1 GB
+    app.config["MAX_FORM_MEMORY_SIZE"] = 1024 * 1024 * 1024  # 1 GB
 
     for bp in all_blueprints():
         app.register_blueprint(bp)
     # Registered last so /api/* routes win over the SPA catch-all.
     app.register_blueprint(frontend_bp)
 
-    @app.before_request
-    async def _extension_cors_preflight():
-        # Short-circuit OPTIONS preflight from the browser extension with a
-        # 204 + CORS headers (the after_request hook below adds them).
-        # Without this, OPTIONS lands on routes that only declared POST/GET
-        # methods and 405s before the after_request gets a chance.
-        if request.method != "OPTIONS":
-            return None
-        origin = request.headers.get("Origin", "")
-        if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
-            return "", 204
-        return None
-
-    @app.after_request
-    async def _extension_cors_headers(response):
-        origin = request.headers.get("Origin", "")
-        if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
-            response.headers["Access-Control-Allow-Origin"] = origin
-            response.headers["Access-Control-Allow-Methods"] = (
-                "GET, POST, PATCH, DELETE, OPTIONS"
-            )
-            response.headers["Access-Control-Allow-Headers"] = (
-                "Content-Type, X-Extension-Key"
-            )
-            response.headers["Access-Control-Max-Age"] = "86400"
-        return response
-
     @app.after_serving
     async def _dispose_db_engine() -> None:
         from .extensions import dispose_engine

backend/app/api/__init__.py

@@ -16,18 +16,17 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"])
 def all_blueprints() -> list[Blueprint]:
     from .admin import admin_bp
     from .aliases import aliases_bp
+    from .allowlist import allowlist_bp
     from .artist import artist_bp
     from .artists import artists_bp
     from .attachments import attachments_bp
-    from .ccip import ccip_bp
     from .cleanup import cleanup_bp
     from .credentials import credentials_bp
     from .downloads import downloads_bp
     from .extension import extension_bp
     from .gallery import gallery_bp
-    from .gpu import gpu_bp
-    from .heads import heads_bp
     from .import_admin import import_admin_bp
+    from .migrate import migrate_bp
     from .ml_admin import ml_admin_bp
     from .platforms import platforms_bp
     from .posts import posts_bp
@@ -55,11 +54,10 @@ def all_blueprints() -> list[Blueprint]:
         admin_bp,
         cleanup_bp,
         import_admin_bp,
+        migrate_bp,
         suggestions_bp,
+        allowlist_bp,
         aliases_bp,
-        heads_bp,
-        gpu_bp,
-        ccip_bp,
         ml_admin_bp,
         thumbnails_bp,
         sources_bp,

backend/app/api/_responses.py

@@ -1,16 +0,0 @@
-"""Shared API response helpers."""
-
-from quart import jsonify
-
-
-def error_response(
-    error: str, *, status: int = 400, detail: str | None = None, **extra,
-):
-    """JSON error body + HTTP status. `detail` is included only when given;
-    `extra` keys are merged into the body. Returns the (response, status)
-    tuple Quart expects. Imported as `_bad` by the blueprints."""
-    body = {"error": error}
-    if detail is not None:
-        body["detail"] = detail
-    body.update(extra)
-    return jsonify(body), status

backend/app/api/admin.py

@@ -1,13 +1,11 @@
 """FC-3k: /api/admin — destructive admin actions.
 
-Action surfaces:
+Five action surfaces:
   POST   /api/admin/artists/<slug>/cascade-delete  (Tier C)
   POST   /api/admin/images/bulk-delete             (Tier C)
   DELETE /api/admin/tags/<int:tag_id>              (Tier B)
   POST   /api/admin/tags/<int:dest_id>/merge       (Tier B)
   POST   /api/admin/tags/prune-unused              (Tier A)
-  POST   /api/admin/posts/prune-bare               (Tier A)
-  POST   /api/admin/posts/refetch-external         (Tier A)
   GET    /api/admin/tags/<int:tag_id>/usage-count  (helper)
 
 Tier-C ops take a dry_run body flag (returns projection inline,
@@ -20,16 +18,21 @@ from __future__ import annotations
 import hashlib
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import select, text
+from sqlalchemy import select
 
 from ..extensions import get_session
-from ..models import Artist, Post
+from ..models import Artist
 from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
-from ._responses import error_response as _bad
 
 admin_bp = Blueprint("admin", __name__, url_prefix="/api/admin")
 
 
+def _bad(error: str, *, status: int = 400, **extra):
+    body = {"error": error}
+    body.update(extra)
+    return jsonify(body), status
+
+
 def _bulk_image_confirm_token(image_ids: list[int]) -> str:
     """Stable 8-hex token derived from the sorted id list. Mutates
     when the selection changes; stays the same across modal opens of
@@ -39,31 +42,6 @@ def _bulk_image_confirm_token(image_ids: list[int]) -> str:
     return digest[:8]
 
 
-async def _run_dry_run_op(service_fn, **service_kwargs):
-    """Shared body for the Tier-A dry-run/apply endpoints: read the `dry_run`
-    flag, run the cleanup_service predicate under `run_sync`, and return its
-    result dict. The SAME `service_fn` drives both preview and apply (the flag
-    just toggles), so a handler physically can't let its preview diverge from
-    its delete (rule 93). Default False preserves the existing contract — the UI
-    always passes `dry_run` explicitly (true to preview, false to apply). Extra
-    service kwargs (e.g. `source_id`) pass straight through."""
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", False))
-    async with get_session() as session:
-        result = await session.run_sync(
-            lambda sync_sess: service_fn(sync_sess, dry_run=dry_run, **service_kwargs)
-        )
-    return jsonify(result)
-
-
-def _queued(async_result):
-    """Standard 202 for an operator-triggered maintenance task: hand the UI the
-    Celery task id so it can tail /maintenance/task-result (or the activity
-    dashboard) for the summary. (trigger_vacuum stays bespoke — the UI doesn't
-    poll it, so it returns no task id.)"""
-    return jsonify({"task_id": async_result.id, "status": "queued"}), 202
-
-
 @admin_bp.route("/artists/<slug>/cascade-delete", methods=["POST"])
 async def artist_cascade_delete(slug: str):
     body = await request.get_json(silent=True) or {}
@@ -119,19 +97,11 @@ async def images_bulk_delete():
             )
         )
 
-    sha8 = _bulk_image_confirm_token(image_ids)
-    expected = f"delete-images-{sha8}"
-
     if dry_run:
-        # Hand the canonical Tier-C confirm token back with the
-        # projection so the frontend doesn't have to recompute SHA-256
-        # client-side via crypto.subtle (Secure-Context-gated,
-        # undefined on plain-HTTP origins per the homelab posture).
-        # Operator-flagged 2026-05-27.
-        projected["confirm_token"] = expected
         return jsonify(projected)
 
-
+    sha8 = _bulk_image_confirm_token(image_ids)
+    expected = f"delete-images-{sha8}"
     if supplied_confirm != expected:
         return _bad(
             "confirm_mismatch",
@@ -156,10 +126,6 @@ async def tag_delete(tag_id: int):
             )
         except LookupError:
             return _bad("not_found", status=404)
-        except ValueError as exc:
-            # System tags (#128) — the training-hygiene machinery keys on
-            # these rows.
-            return _bad("system_tag", detail=str(exc))
     return jsonify(result)
 
 
@@ -175,30 +141,6 @@ async def tag_merge(dest_id: int):
     if not isinstance(source_id, int) or source_id == dest_id:
         return _bad("invalid_source_id", detail="source_id must be int and differ from dest")
 
-    # dry_run: non-mutating preview (counts + sample) so the operator can
-    # confirm the target before the irreversible merge (#8, rule 93 parity).
-    if body.get("dry_run"):
-        async with get_session() as session:
-            try:
-                p = await TagService(session).merge_preview(
-                    source_id=source_id, target_id=dest_id,
-                )
-            except TagValidationError as exc:
-                return _bad("tag_not_found", status=404, detail=str(exc))
-        return jsonify({
-            "preview": {
-                "source_id": p.source_id, "source_name": p.source_name,
-                "target_id": p.target_id, "target_name": p.target_name,
-                "compatible": p.compatible,
-                "images_moving": p.images_moving,
-                "images_already_on_target": p.images_already_on_target,
-                "source_total": p.source_total,
-                "series_pages": p.series_pages,
-                "will_alias": p.will_alias,
-                "sample_thumbnails": p.sample_thumbnails,
-            },
-        })
-
     async with get_session() as session:
         try:
             result = await TagService(session).merge(
@@ -246,261 +188,13 @@ async def tags_prune_unused():
     re-call with dry_run=false."""
     from ..services.cleanup_service import prune_unused_tags
 
-    return await _run_dry_run_op(prune_unused_tags)
-
-
-@admin_bp.route("/posts/prune-bare", methods=["POST"])
-async def posts_prune_bare():
-    """Tier-A: delete bare posts — Post rows with no linked images (primary OR
-    provenance) and no attachments. Dry-run preview list IS the prompt: UI calls
-    with dry_run=true first, shows the count + sample, operator confirms by
-    re-calling with dry_run=false. Same preview/apply-parity predicate as the
-    prune itself, so the preview can't diverge from the delete."""
-    from ..services.cleanup_service import prune_bare_posts
-
-    return await _run_dry_run_op(prune_bare_posts)
-
-
-@admin_bp.route("/posts/reconcile-duplicates", methods=["POST"])
-async def posts_reconcile_duplicates():
-    """Tier-A: unify duplicate post rows for the same real post — the gallery-dl
-    (attachment-id) + native (post-id) duplicates — onto ONE post-id-keyed keeper,
-    moving image/provenance/attachment/link rows over. Images are untouched.
-    dry_run=true returns {groups, posts_to_merge, sample}; dry_run=false applies
-    and returns {groups, merged, sample}. Optional source_id scopes to one source.
-    Same find_duplicate_post_groups predicate drives preview + apply (rule 93)."""
-    from ..services.cleanup_service import reconcile_duplicate_posts
-
-    body = await request.get_json(silent=True) or {}
-    raw_source = body.get("source_id")
-    try:
-        source_id = int(raw_source) if raw_source is not None else None
-    except (TypeError, ValueError):
-        return _bad("invalid_source_id", detail="source_id must be an integer")
-
-    return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
-
-
-@admin_bp.route("/posts/refetch-external", methods=["POST"])
-async def posts_refetch_external():
-    """Surgical re-fetch of a post's external file-host links (operator
-    2026-07-03): the normal cadence never re-walks deep back-catalogue posts,
-    so a deleted external file only comes back by resetting its ExternalLink
-    row(s) — this endpoint does that per post and dispatches the fetches.
-    Sha-dedupe discards payload files that still exist, so only what's
-    missing lands. Body: {external_post_id: str, source_id?: int (to
-    disambiguate the same external id across sources)}."""
-    from ..services.external_links import refetch_links_for_post
-
-    body = await request.get_json(silent=True) or {}
-    ext_id = str(body.get("external_post_id") or "").strip()
-    if not ext_id:
-        return _bad("missing_external_post_id",
-                    detail="external_post_id is required")
-    raw_source = body.get("source_id")
-    try:
-        source_id = int(raw_source) if raw_source is not None else None
-    except (TypeError, ValueError):
-        return _bad("invalid_source_id", detail="source_id must be an integer")
-
-    async with get_session() as session:
-        stmt = select(Post.id).where(Post.external_post_id == ext_id)
-        if source_id is not None:
-            stmt = stmt.where(Post.source_id == source_id)
-        post_ids = (await session.execute(stmt)).scalars().all()
-        if not post_ids:
-            return _bad("post_not_found", status=404,
-                        detail=f"no post with external_post_id {ext_id!r}")
-        results = {}
-        for pid in post_ids:
-            results[str(pid)] = await session.run_sync(
-                lambda s, p=pid: refetch_links_for_post(s, p)
-            )
-    return jsonify({"posts": results})
-
-
-def _reset_content_confirm_token(projection: dict) -> str:
-    """Stable 8-hex token derived from the live counts (mirrors the Tier-C
-    bulk-delete token): it changes whenever the data changes, so the apply can
-    only ever run against numbers the operator just previewed."""
-    canon = f"reset-content:{projection.get('count')}:{projection.get('applications')}"
-    return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:8]
-
-
-@admin_bp.route("/tags/reset-content", methods=["POST"])
-async def tags_reset_content():
-    """Full-instance reset of the CONTENT vocabulary: deletes ALL general +
-    character tags and their image applications — INCLUDING the examples the
-    tagging heads learned from. Suggestions do NOT repopulate on their own
-    (the Camie predictions that once did are long retired): the operator
-    re-tags from scratch and the heads retrain from the new signal. fandom +
-    series tags + series_page ordering are preserved.
-
-    Deliberately Tier-C-gated despite the Tier-A shape (operator 2026-07-02:
-    the full reset stays, but behind extra steps): dry_run returns the
-    projection + a `confirm` token derived from the live counts; the apply
-    must echo that token back or it is rejected."""
-    from ..services.cleanup_service import reset_content_tagging
-
     body = await request.get_json(silent=True) or {}
     dry_run = bool(body.get("dry_run", False))
+
     async with get_session() as session:
-        projection = await session.run_sync(
-            lambda s: reset_content_tagging(s, dry_run=True)
-        )
-        token = _reset_content_confirm_token(projection)
-        if dry_run:
-            projection["confirm"] = token
-            return jsonify(projection)
-        if str(body.get("confirm", "")) != token:
-            return _bad(
-                "confirm_mismatch",
-                detail="run a fresh preview and echo its confirm token",
-            )
         result = await session.run_sync(
-            lambda s: reset_content_tagging(s, dry_run=False)
+            lambda sync_sess: prune_unused_tags(
+                sync_sess, dry_run=dry_run,
+            )
         )
     return jsonify(result)
-
-
-@admin_bp.route("/tags/normalize", methods=["POST"])
-async def tags_normalize():
-    """#714: retro-normalize existing tags to the #701 canonical form (Title
-    Case + collapsed whitespace) and merge case/whitespace-variant duplicates.
-
-    dry_run=true (default) returns a projection inline — group/collision/rename
-    counts + a sample of the changes — so the UI shows exactly what'll happen.
-    dry_run=false dispatches the long-running maintenance task (the merge FK
-    repoints can touch many tags); the UI tails the activity dashboard for the
-    summary. Idempotent; back up first (the merges are irreversible)."""
-    from ..services.tag_service import normalize_existing_tags
-
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", True))
-
-    if dry_run:
-        async with get_session() as session:
-            result = await normalize_existing_tags(session, dry_run=True)
-        return jsonify(result)
-
-    from ..tasks.admin import normalize_tags_task
-
-    async_result = normalize_tags_task.delay()
-    return _queued(async_result)
-
-
-@admin_bp.route("/maintenance/db-stats", methods=["GET"])
-async def db_stats():
-    """Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
-    so the operator can see when a VACUUM is worth running."""
-    from ..tasks.maintenance import VACUUM_TABLES
-
-    wanted = set(VACUUM_TABLES)
-    async with get_session() as session:
-        rows = (await session.execute(text(
-            "SELECT relname, n_live_tup, n_dead_tup, last_vacuum, "
-            "last_autovacuum, last_analyze FROM pg_stat_user_tables"
-        ))).all()
-
-    def _iso(v):
-        return v.isoformat() if v is not None else None
-
-    out = []
-    for r in rows:
-        if r.relname not in wanted:
-            continue
-        live = r.n_live_tup or 0
-        dead = r.n_dead_tup or 0
-        total = live + dead
-        out.append({
-            "table": r.relname,
-            "live": live,
-            "dead": dead,
-            "dead_pct": round(100 * dead / total, 1) if total else 0.0,
-            "last_vacuum": _iso(r.last_vacuum),
-            "last_autovacuum": _iso(r.last_autovacuum),
-            "last_analyze": _iso(r.last_analyze),
-        })
-    out.sort(key=lambda t: t["dead"], reverse=True)
-    return jsonify({"tables": out})
-
-
-@admin_bp.route("/maintenance/vacuum", methods=["POST"])
-async def trigger_vacuum():
-    """Operator-triggered VACUUM (ANALYZE) over the high-churn tables — the
-    same maintenance-queue task the weekly Beat schedule runs."""
-    from ..tasks.maintenance import vacuum_analyze
-
-    vacuum_analyze.delay()
-    return jsonify({"status": "queued"}), 202
-
-
-@admin_bp.route("/maintenance/reextract-archives", methods=["POST"])
-async def trigger_reextract_archives():
-    """Operator-triggered re-extract (#713): PostAttachments that are actually
-    archives but were filed opaquely (pre magic-byte gate) get extracted and
-    their members linked to the post. Idempotent; runs on the maintenance queue."""
-    from ..tasks.admin import reextract_archive_attachments_task
-
-    async_result = reextract_archive_attachments_task.delay()
-    return _queued(async_result)
-
-
-@admin_bp.route("/maintenance/prune-missing-files", methods=["POST"])
-async def trigger_prune_missing_files():
-    """Operator-triggered orphan repair (#859): delete ImageRecords whose backing
-    file is gone from disk (e.g. left by the external-attach unlink bug), so they
-    stop 404-ing on playback. The task aborts WITHOUT deleting if a large fraction
-    of files look missing (a filesystem/NFS stall). Maintenance queue;
-    operator-triggered only — never an unattended sweep."""
-    from ..tasks.admin import prune_missing_file_records_task
-
-    async_result = prune_missing_file_records_task.delay()
-    return _queued(async_result)
-
-
-@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
-async def trigger_dedup_videos():
-    """Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
-    what would be removed (groups / redundant count / reclaimable bytes) WITHOUT
-    deleting; dry_run=false applies it (re-link posts to the keeper, then delete
-    the redundant copies). Either way it first re-probes NULL-duration videos so
-    the existing library participates. Returns the Celery task id — poll
-    /maintenance/task-result/<id> for the summary."""
-    from ..tasks.admin import dedup_videos_task
-
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", True))  # default to the SAFE preview
-    async_result = dedup_videos_task.delay(dry_run=dry_run)
-    return _queued(async_result)
-
-
-@admin_bp.route("/maintenance/purge-gated-previews", methods=["POST"])
-async def trigger_purge_gated_previews():
-    """Cleanup (#874 follow-up). Body {"dry_run": bool}: dry_run=true previews how
-    many blurred locked-preview images (grabbed from tier-gated Patreon posts
-    before the fix) would be removed WITHOUT deleting; dry_run=false applies it.
-    Re-walks every enabled Patreon source read-only and matches by content hash, so
-    real content downloaded when access existed is provably spared. Returns the
-    Celery task id — poll /maintenance/task-result/<id> for the summary."""
-    from ..tasks.admin import purge_gated_previews_task
-
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", True))  # default to the SAFE preview
-    async_result = purge_gated_previews_task.delay(dry_run=dry_run)
-    return _queued(async_result)
-
-
-@admin_bp.route("/maintenance/task-result/<task_id>", methods=["GET"])
-async def maintenance_task_result(task_id: str):
-    """Poll a maintenance Celery task's result (the summary dict it returns).
-    Used by the video-dedup card to show the dry-run projection before apply."""
-    from ..celery_app import celery
-
-    res = celery.AsyncResult(task_id)
-    ready = res.ready()
-    return jsonify({
-        "ready": ready,
-        "successful": res.successful() if ready else None,
-        "result": res.result if (ready and res.successful()) else None,
-    })

backend/app/api/allowlist.py

@@ -0,0 +1,59 @@
+"""Allowlist API: list, adjust threshold, remove."""
+
+from quart import Blueprint, jsonify, request
+
+from ..extensions import get_session
+from ..models import TagAllowlist
+from ..services.ml.allowlist import AllowlistService
+
+allowlist_bp = Blueprint("allowlist", __name__, url_prefix="/api")
+
+
+@allowlist_bp.route("/allowlist", methods=["GET"])
+async def list_allowlist():
+    async with get_session() as session:
+        rows = await AllowlistService(session).list_all()
+    return jsonify(
+        [
+            {
+                "tag_id": r.tag_id,
+                "tag_name": r.tag_name,
+                "tag_kind": r.tag_kind,
+                "min_confidence": r.min_confidence,
+            }
+            for r in rows
+        ]
+    )
+
+
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["GET"])
+async def get_one(tag_id: int):
+    async with get_session() as session:
+        row = await session.get(TagAllowlist, tag_id)
+    if row is None:
+        return jsonify({"error": "not on allowlist"}), 404
+    return jsonify(
+        {"min_confidence": row.min_confidence, "added_at": row.added_at.isoformat()}
+    )
+
+
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["PATCH"])
+async def patch_threshold(tag_id: int):
+    body = await request.get_json()
+    if not body or "min_confidence" not in body:
+        return jsonify({"error": "min_confidence required"}), 400
+    mc = float(body["min_confidence"])
+    if not (0 < mc <= 1):
+        return jsonify({"error": "min_confidence must be in (0, 1]"}), 400
+    async with get_session() as session:
+        await AllowlistService(session).update_threshold(tag_id, mc)
+        await session.commit()
+    return "", 204
+
+
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["DELETE"])
+async def remove(tag_id: int):
+    async with get_session() as session:
+        await AllowlistService(session).remove(tag_id)
+        await session.commit()
+    return "", 204

backend/app/api/artists.py

@@ -31,24 +31,6 @@ async def create_or_get():
     }), 201
 
 
-@artists_bp.route("/<int:artist_id>", methods=["PATCH"])
-async def rename(artist_id: int):
-    """Rename an artist's DISPLAY NAME (#130). Name only — the slug and every
-    on-disk path stay put, so this is instant and safe. Name is non-unique."""
-    body = await request.get_json()
-    if not isinstance(body, dict) or not isinstance(body.get("name"), str):
-        return jsonify({"error": "invalid_body"}), 400
-    async with get_session() as session:
-        svc = ArtistService(session)
-        try:
-            artist = await svc.rename(artist_id, body["name"])
-        except ValueError as exc:
-            return jsonify({"error": "empty_name", "detail": str(exc)}), 400
-        if artist is None:
-            return jsonify({"error": "not_found"}), 404
-        return jsonify({"id": artist.id, "name": artist.name, "slug": artist.slug})
-
-
 @artists_bp.route("/autocomplete", methods=["GET"])
 async def autocomplete():
     q = request.args.get("q") or ""

backend/app/api/ccip.py

@@ -1,124 +0,0 @@
-"""CCIP / region observability API (#114) — read-only, analysis-shaped.
-
-So the work can be checked through an API as the agent fills in vectors: overall
-coverage (regions by kind, how many images have figure CCIP vectors, which
-characters have enough reference examples to match on) + a per-image drill-down
-(its regions + the CCIP character matches it would get). Mirrors the heads
-metrics endpoint; no GPU, just reads what's stored.
-"""
-
-from quart import Blueprint, jsonify
-from sqlalchemy import distinct, func, select
-
-from ..extensions import get_session
-from ..models import ImageRegion, Tag, TagKind
-from ..models.tag import image_tag
-from ..services.ml.ccip import match_image
-
-ccip_bp = Blueprint("ccip", __name__, url_prefix="/api/ccip")
-
-_FIGURE_KINDS = ("face", "figure")
-
-
-@ccip_bp.route("/overview", methods=["GET"])
-async def overview():
-    async with get_session() as session:
-        by_kind = dict(
-            (
-                await session.execute(
-                    select(ImageRegion.kind, func.count()).group_by(ImageRegion.kind)
-                )
-            ).all()
-        )
-        images_with_figure_ccip = (
-            await session.execute(
-                select(func.count(distinct(ImageRegion.image_record_id)))
-                .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-                .where(ImageRegion.ccip_embedding.is_not(None))
-            )
-        ).scalar_one()
-        # Concept-crop (SigLIP bag) coverage — how far the back-catalogue embed
-        # has progressed, so the max-over-bag scorer's reach is checkable.
-        images_with_concept_siglip = (
-            await session.execute(
-                select(func.count(distinct(ImageRegion.image_record_id)))
-                .where(ImageRegion.kind == "concept")
-                .where(ImageRegion.siglip_embedding.is_not(None))
-            )
-        ).scalar_one()
-        # Per-character reference counts (no vectors loaded) — which characters
-        # have enough examples to match on.
-        ref_rows = (
-            await session.execute(
-                select(image_tag.c.tag_id, Tag.name, func.count())
-                .select_from(ImageRegion)
-                .join(
-                    image_tag,
-                    image_tag.c.image_record_id == ImageRegion.image_record_id,
-                )
-                .join(Tag, Tag.id == image_tag.c.tag_id)
-                .where(Tag.kind == TagKind.character)
-                .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-                .where(ImageRegion.ccip_embedding.is_not(None))
-                .group_by(image_tag.c.tag_id, Tag.name)
-                .order_by(func.count().desc())
-            )
-        ).all()
-        versions = [
-            v for (v,) in (
-                await session.execute(
-                    select(distinct(ImageRegion.embedding_version))
-                )
-            ).all() if v
-        ]
-        auto_applied = (
-            await session.execute(
-                select(func.count()).select_from(image_tag).where(
-                    image_tag.c.source == "ccip_auto"
-                )
-            )
-        ).scalar_one()
-    return jsonify({
-        "regions_by_kind": by_kind,
-        "images_with_figure_ccip": images_with_figure_ccip,
-        "images_with_concept_siglip": images_with_concept_siglip,
-        "characters_with_references": len(ref_rows),
-        "character_references": [
-            {"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows
-        ],
-        "embedding_versions": versions,
-        "auto_applied": auto_applied,
-    })
-
-
-@ccip_bp.route("/images/<int:image_id>", methods=["GET"])
-async def image_detail(image_id: int):
-    """An image's stored regions + the CCIP character matches it would get —
-    for spot-checking the agent's output + the matcher."""
-    async with get_session() as session:
-        regions = (
-            await session.execute(
-                select(ImageRegion)
-                .where(ImageRegion.image_record_id == image_id)
-                .order_by(ImageRegion.id)
-            )
-        ).scalars().all()
-        matches = await match_image(session, image_id)
-    return jsonify({
-        "image_id": image_id,
-        "regions": [
-            {
-                "id": r.id,
-                "kind": r.kind,
-                "bbox": [r.rx, r.ry, r.rw, r.rh],
-                "frame_time": r.frame_time,
-                "score": r.score,
-                "detector_version": r.detector_version,
-                "embedding_version": r.embedding_version,
-                "has_ccip": r.ccip_embedding is not None,
-                "has_siglip": r.siglip_embedding is not None,
-            }
-            for r in regions
-        ],
-        "ccip_matches": matches,
-    })

backend/app/api/cleanup.py

@@ -31,13 +31,18 @@ from sqlalchemy import select
 from ..extensions import get_session
 from ..models import LibraryAuditRun
 from ..services import cleanup_service
-from ._responses import error_response as _bad
 
 cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup")
 
 IMAGES_ROOT = Path("/images")
 
 
+def _bad(error: str, *, status: int = 400, **extra):
+    body = {"error": error}
+    body.update(extra)
+    return jsonify(body), status
+
+
 def _min_dim_token(min_w: int, min_h: int) -> str:
     # SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both
     # sides use SHA-256 truncated to 8 hex chars.
@@ -76,13 +81,6 @@ async def min_dim_preview():
                 s, min_width=min_w, min_height=min_h,
             )
         )
-    # Hand the canonical Tier-C delete token back with the preview so
-    # the frontend doesn't have to recompute SHA-256 client-side.
-    # window.crypto.subtle is Secure-Context-gated and undefined on
-    # plain-HTTP origins (homelab posture); without this the Delete
-    # button silently swallowed the TypeError and never opened the
-    # confirm modal. Operator-flagged 2026-05-27.
-    projection["confirm_token"] = _min_dim_token(min_w, min_h)
     return jsonify(projection)
 
 
@@ -154,15 +152,12 @@ async def audit_history():
         limit = min(int(request.args.get("limit", "20")), 100)
     except ValueError:
         return _bad("invalid_limit")
-    # Optional rule filter so a card can reconnect to ITS latest run on mount
-    # (?rule=transparency&limit=1) — the audit survives navigation; the UI
-    # rehydrates from this rather than losing the in-flight scan.
-    rule = request.args.get("rule") or None
     async with get_session() as session:
-        stmt = select(LibraryAuditRun).order_by(LibraryAuditRun.id.desc())
-        if rule is not None:
-            stmt = stmt.where(LibraryAuditRun.rule == rule)
-        rows = (await session.execute(stmt.limit(limit))).scalars().all()
+        rows = (await session.execute(
+            select(LibraryAuditRun)
+            .order_by(LibraryAuditRun.id.desc())
+            .limit(limit)
+        )).scalars().all()
         return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
 
 

backend/app/api/credentials.py

@@ -20,7 +20,6 @@ from ..services.credential_service import (
     UnknownPlatformError,
     WrongAuthTypeError,
 )
-from ._responses import error_response as _bad
 
 credentials_bp = Blueprint("credentials", __name__, url_prefix="/api/credentials")
 
@@ -39,6 +38,14 @@ def _get_crypto() -> CredentialCrypto:
     return _crypto
 
 
+def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
+    body = {"error": error}
+    if detail is not None:
+        body["detail"] = detail
+    body.update(extra)
+    return jsonify(body), status
+
+
 async def _ext_key_ok(session) -> bool:
     """If X-Extension-Key is supplied, it must match the stored value.
     Missing header → True (browser path; accepted per homelab posture).
@@ -117,58 +124,3 @@ async def delete_credential(platform: str):
         except LookupError:
             return _bad("not_found", status=404)
     return "", 204
-
-
-@credentials_bp.route("/<platform>/verify", methods=["POST"])
-async def verify_credential(platform: str):
-    """Test the stored credential against one of the platform's enabled sources,
-    WITHOUT downloading. Routes through the platform's backend
-    (download_backends.verify_credential) — native ingester for Patreon, an
-    authenticated API page; gallery-dl --simulate for the rest. On success
-    stamps last_verified. Returns {valid: bool|null, reason, last_verified?};
-    valid=null means "couldn't test" (no credential, no enabled source, or an
-    inconclusive network/drift result)."""
-    from ..models import Artist, Source
-    from ..services.download_backends import verify_source_credential
-
-    async with get_session() as session:
-        if not await _ext_key_ok(session):
-            return _bad("unauthorized", status=401)
-        svc = CredentialService(session, _get_crypto())
-        record = await svc.get(platform)
-        if record is None:
-            return jsonify({"valid": None, "reason": "No credential stored for this platform."})
-
-        # Pick an enabled source for this platform to point the probe at.
-        row = (await session.execute(
-            select(Source, Artist)
-            .join(Artist, Artist.id == Source.artist_id)
-            .where(Source.platform == platform, Source.enabled.is_(True))
-            .order_by(Source.id.asc())
-        )).first()
-        if row is None:
-            return jsonify({
-                "valid": None,
-                "reason": "No enabled source for this platform to verify against — add a subscription first.",
-            })
-        source, artist = row
-
-        cookies_path = await svc.get_cookies_path(platform)
-        auth_token = await svc.get_token(platform)
-
-    ok, message = await verify_source_credential(
-        platform=platform,
-        url=source.url,
-        artist_slug=artist.slug,
-        config_overrides=source.config_overrides or {},
-        cookies_path=str(cookies_path) if cookies_path else None,
-        auth_token=auth_token,
-        images_root=Path("/images"),
-    )
-
-    last_verified = None
-    if ok:
-        async with get_session() as session:
-            ts = await CredentialService(session, _get_crypto()).mark_verified(platform)
-            last_verified = ts.isoformat() if ts else None
-    return jsonify({"valid": ok, "reason": message, "last_verified": last_verified})

backend/app/api/downloads.py

@@ -5,10 +5,8 @@ status/source/artist. Returns slim records.
 Detail view: full DownloadEvent including the metadata JSONB.
 """
 
-from datetime import UTC, datetime, timedelta
-
 from quart import Blueprint, jsonify, request
-from sqlalchemy import func, select
+from sqlalchemy import select
 
 from ..extensions import get_session
 from ..models import Artist, DownloadEvent, Source
@@ -44,9 +42,6 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N
         "bytes_downloaded": event.bytes_downloaded,
         "error": event.error,
         "summary": _summary_from_metadata(event.metadata_),
-        # plan #709: mid-walk live counts for a RUNNING native-ingester event
-        # (None otherwise; phase 3 overwrites metadata with run_stats on finish).
-        "live": (event.metadata_ or {}).get("live"),
     }
 
 
@@ -100,83 +95,6 @@ async def list_downloads():
     return jsonify([_list_record(e, s, a) for e, s, a in rows])
 
 
-@downloads_bp.route("/stats", methods=["GET"])
-async def downloads_stats():
-    """Status-grouped count over download_event for the dashboard stat chips.
-
-    `?window_hours=` (default 24) bounds by `started_at`. The full set of
-    statuses is always present in the response (zero for missing) so the
-    UI doesn't have to fill in defaults.
-    """
-    try:
-        window_hours = int(request.args.get("window_hours", "24"))
-    except ValueError:
-        return jsonify({"error": "invalid_window_hours"}), 400
-    if window_hours < 1 or window_hours > 24 * 365:
-        return jsonify({"error": "invalid_window_hours"}), 400
-
-    since = datetime.now(UTC) - timedelta(hours=window_hours)
-    out = {"pending": 0, "running": 0, "ok": 0, "error": 0, "skipped": 0}
-    async with get_session() as session:
-        stmt = (
-            select(DownloadEvent.status, func.count())
-            .where(DownloadEvent.started_at >= since)
-            .group_by(DownloadEvent.status)
-        )
-        for status, n in (await session.execute(stmt)).all():
-            if status in out:
-                out[status] = int(n)
-    return jsonify(out)
-
-
-@downloads_bp.route("/activity", methods=["GET"])
-async def downloads_activity():
-    """Hourly download-event counts over the last `?hours=` (default 24).
-
-    Returns a fixed-length, oldest-first bucket array so the UI can render
-    a sparkline directly. Bucketing is done in Python against UTC to dodge
-    session-timezone ambiguity in SQL date_trunc.
-    """
-    try:
-        hours = int(request.args.get("hours", "24"))
-    except ValueError:
-        return jsonify({"error": "invalid_hours"}), 400
-    hours = max(1, min(168, hours))
-
-    now = datetime.now(UTC)
-    end = now.replace(minute=0, second=0, microsecond=0)
-    start = end - timedelta(hours=hours - 1)
-    buckets = [
-        {"hour": (start + timedelta(hours=i)).isoformat(),
-         "ok": 0, "error": 0, "other": 0, "total": 0}
-        for i in range(hours)
-    ]
-
-    async with get_session() as session:
-        rows = (await session.execute(
-            select(DownloadEvent.started_at, DownloadEvent.status)
-            .where(DownloadEvent.started_at >= start)
-        )).all()
-
-    for started_at, status in rows:
-        if started_at is None:
-            continue
-        sa = started_at if started_at.tzinfo else started_at.replace(tzinfo=UTC)
-        idx = int((sa - start).total_seconds() // 3600)
-        if not (0 <= idx < hours):
-            continue
-        b = buckets[idx]
-        if status == "ok":
-            b["ok"] += 1
-        elif status == "error":
-            b["error"] += 1
-        else:
-            b["other"] += 1
-        b["total"] += 1
-
-    return jsonify({"hours": hours, "buckets": buckets})
-
-
 @downloads_bp.route("/<int:event_id>", methods=["GET"])
 async def get_download(event_id: int):
     async with get_session() as session:
@@ -190,20 +108,3 @@ async def get_download(event_id: int):
             return jsonify({"error": "not_found"}), 404
         event, source, artist = row
     return jsonify(_detail_record(event, source, artist))
-
-
-@downloads_bp.route("/recover-stalled", methods=["POST"])
-async def recover_stalled():
-    """Trigger the recover_stalled_download_events sweep on demand.
-
-    The same sweep runs every 5 min via Beat (see celery_app.beat_schedule);
-    this endpoint exists so the operator can force-clear stuck pending/
-    running download_events from the Subscriptions → Downloads maintenance
-    menu without waiting for the next scheduled tick.
-    """
-    # Local import: avoids registering maintenance tasks during blueprint
-    # import (Celery task discovery races with the API import otherwise).
-    from ..tasks.maintenance import recover_stalled_download_events
-
-    recover_stalled_download_events.delay()
-    return jsonify({"queued": True}), 202

backend/app/api/extension.py

@@ -20,7 +20,6 @@ from ..services.extension_service import (
     UnknownPlatformError,
 )
 from ..services.source_service import KNOWN_PLATFORMS
-from ._responses import error_response as _bad
 
 extension_bp = Blueprint("extension", __name__, url_prefix="/api/extension")
 
@@ -31,6 +30,12 @@ XPI_DIR = Path("/app/frontend/dist/extension")
 _XPI_VERSION_RE = re.compile(r"fabledcurator-(?P<version>[\w.-]+)\.xpi$")
 
 
+def _bad(error: str, *, status: int = 400, **extra):
+    body = {"error": error}
+    body.update(extra)
+    return jsonify(body), status
+
+
 async def _ext_key_required(session) -> bool:
     """Unlike /api/credentials (which accepts the browser path with no
     header), quick-add-source writes server state and must be explicitly
@@ -57,24 +62,6 @@ def _sha256(path: Path) -> str:
     return h.hexdigest()
 
 
-@extension_bp.route("/probe", methods=["GET"])
-async def probe_source():
-    """Read-only resolution of a creator-page URL: tells the extension
-    whether this URL is already a Source, is for an Artist that exists
-    but with a different URL, is brand new, or doesn't match any known
-    platform pattern. Drives the content-script chip's color/copy
-    BEFORE the operator clicks, so the button can show 'already added'
-    without requiring an add-attempt."""
-    url = (request.args.get("url") or "").strip()
-    if not url:
-        return _bad("invalid_body", detail="url query parameter is required")
-    async with get_session() as session:
-        if not await _ext_key_required(session):
-            return _bad("unauthorized", status=401)
-        result = await ExtensionService(session).probe(url)
-    return jsonify(result)
-
-
 @extension_bp.route("/quick-add-source", methods=["POST"])
 async def quick_add_source():
     body = await request.get_json(silent=True)
@@ -84,15 +71,11 @@ async def quick_add_source():
     if not isinstance(url, str) or not url.strip():
         return _bad("invalid_body", detail="url is required")
 
-    from .credentials import _get_crypto
-
     async with get_session() as session:
         if not await _ext_key_required(session):
             return _bad("unauthorized", status=401)
         try:
-            # crypto lets a pixiv add resolve the artist's display name via the
-            # stored OAuth token (else it falls back to the numeric id). #130.
-            result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
+            result = await ExtensionService(session).quick_add_source(url)
         except UnknownPlatformError as exc:
             return _bad(
                 "unknown_platform",

backend/app/api/gallery.py

@@ -1,131 +1,54 @@
-"""Gallery API: cursor scroll, timeline, jump, image detail, facets."""
-
-from datetime import UTC, datetime, timedelta
+"""Gallery API: cursor scroll, timeline, jump, image detail."""
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import delete, select, update
-from sqlalchemy.orm import aliased
 
 from ..extensions import get_session
-from ..models import (
-    ImageRecord,
-    PresentationReview,
-    Tag,
-    TagSuggestionRejection,
-)
-from ..models.tag import image_tag
-from ..services.gallery_service import GalleryService, image_url, thumbnail_url
+from ..services.gallery_service import GalleryService
 
 gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
 
 
-def _image_json(i):
-    """Serialize a GalleryImage for the scroll/similar list responses."""
-    return {
-        "id": i.id,
-        "sha256": i.sha256,
-        "mime": i.mime,
-        "width": i.width,
-        "height": i.height,
-        "created_at": i.created_at.isoformat(),
-        "posted_at": i.posted_at.isoformat() if i.posted_at else None,
-        "thumbnail_url": i.thumbnail_url,
-        "artist": i.artist,
-    }
-
-
-def _parse_date(raw):
-    """Parse a YYYY-MM-DD query value to a UTC midnight datetime, or None.
-    Raises ValueError (→ 400) on a malformed value."""
-    if not raw:
-        return None
-    return datetime.strptime(raw, "%Y-%m-%d").replace(tzinfo=UTC)
-
-
-def _parse_filters():
-    """Parse the composable gallery filters from query args, returning
-    ``(filters_dict, sort)``. Raises ValueError (→ 400) on malformed ids/dates.
-
-    The structured tag filter (#6) is AND-of-OR plus exclusions:
-    - `tag_id` accepts a single id or a comma-separated list — all ANDed
-      (the include common case; back-compat).
-    - `tag_or` is REPEATABLE; each instance is a comma-separated OR-group, and
-      the image must match at least one tag from EACH group (groups ANDed).
-    - `tag_not` is a comma-separated exclude list (image must carry none).
-
-    `media` is image|video; `sort` is newest|oldest|posted_new|posted_old
-    (default posted_new); `platform` selects one
-    platform (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are
-    boolean flags; `date_from`/`date_to` are inclusive calendar-day bounds
-    (date_to is widened by a day so the whole day is covered by the service's
-    half-open `< date_to`)."""
-    tag_raw = request.args.get("tag_id")
-    tag_ids = (
-        [int(x) for x in tag_raw.split(",") if x.strip()] if tag_raw else None
-    ) or None
-    tag_or_groups = [
-        grp for raw in request.args.getlist("tag_or")
-        if (grp := [int(x) for x in raw.split(",") if x.strip()])
-    ] or None
-    not_raw = request.args.get("tag_not")
-    tag_exclude = (
-        [int(x) for x in not_raw.split(",") if x.strip()] if not_raw else None
-    ) or None
-    post_id_raw = request.args.get("post_id")
-    post_id = int(post_id_raw) if post_id_raw else None
-    artist_id_raw = request.args.get("artist_id")
-    artist_id = int(artist_id_raw) if artist_id_raw else None
-    media = request.args.get("media")
-    media_type = media if media in ("image", "video") else None
-    # newest/oldest key off effective_date (primary post / download); posted_new/
-    # posted_old off earliest_post_date (original publish across all posts). The
-    # default is posted_new so the grid leads with original publish date, not the
-    # download/repost the primary post points at (operator-flagged 2026-07-01).
-    sort = request.args.get("sort")
-    sort = sort if sort in ("newest", "oldest", "posted_new", "posted_old") else "posted_new"
-    platform = request.args.get("platform") or None
-    untagged = request.args.get("untagged") in ("1", "true", "yes")
-    no_artist = request.args.get("no_artist") in ("1", "true", "yes")
-    # Show the presentation chrome (banner / editor screenshot) that the default
-    # gallery hides — the Hidden view sets this (milestone 141).
-    include_hidden = request.args.get("include_hidden") in ("1", "true", "yes")
-    date_from = _parse_date(request.args.get("date_from"))
-    date_to = _parse_date(request.args.get("date_to"))
-    if date_to is not None:
-        date_to += timedelta(days=1)  # inclusive of the date_to calendar day
-    filters = {
-        "tag_ids": tag_ids, "post_id": post_id, "artist_id": artist_id,
-        "media_type": media_type,
-        "tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
-        "platform": platform,
-        "untagged": untagged, "no_artist": no_artist,
-        "date_from": date_from, "date_to": date_to,
-        "include_hidden": include_hidden,
-    }
-    return filters, sort
-
-
 @gallery_bp.route("/scroll", methods=["GET"])
 async def scroll():
     cursor = request.args.get("cursor") or None
     try:
         limit = int(request.args.get("limit", "50"))
-        filters, sort = _parse_filters()
     except ValueError:
-        return jsonify({"error": "invalid filter or limit parameter"}), 400
+        return jsonify({"error": "limit must be an integer"}), 400
+    tag_id_raw = request.args.get("tag_id")
+    tag_id = int(tag_id_raw) if tag_id_raw else None
+    post_id_raw = request.args.get("post_id")
+    post_id = int(post_id_raw) if post_id_raw else None
+    artist_id_raw = request.args.get("artist_id")
+    artist_id = int(artist_id_raw) if artist_id_raw else None
 
     async with get_session() as session:
         svc = GalleryService(session)
         try:
             page = await svc.scroll(
-                cursor=cursor, limit=limit, sort=sort, **filters,
+                cursor=cursor, limit=limit, tag_id=tag_id,
+                post_id=post_id, artist_id=artist_id,
             )
         except ValueError as exc:
             return jsonify({"error": str(exc)}), 400
 
     return jsonify(
         {
-            "images": [_image_json(i) for i in page.images],
+            "images": [
+                {
+                    "id": i.id,
+                    "sha256": i.sha256,
+                    "mime": i.mime,
+                    "width": i.width,
+                    "height": i.height,
+                    "created_at": i.created_at.isoformat(),
+                    "posted_at": i.posted_at.isoformat() if i.posted_at else None,
+                    "effective_date": i.effective_date.isoformat(),
+                    "thumbnail_url": i.thumbnail_url,
+                    "artist": i.artist,
+                }
+                for i in page.images
+            ],
             "next_cursor": page.next_cursor,
             "date_groups": [
                 {"year": y, "month": m, "image_ids": ids} for y, m, ids in page.date_groups
@@ -134,50 +57,20 @@ async def scroll():
     )
 
 
-@gallery_bp.route("/similar", methods=["GET"])
-async def similar():
-    """Visual "more like this": images ranked by cosine distance to the
-    `similar_to` image's embedding. Composes with the scope filters (AND) but
-    ignores post_id and sort. Bounded top-N, no cursor."""
-    try:
-        similar_to = int(request.args["similar_to"])
-        limit = int(request.args.get("limit", "100"))
-        filters, _sort = _parse_filters()
-    except (KeyError, ValueError):
-        return jsonify({"error": "similar_to query param required"}), 400
-    # post_id is the exclusive post-detail view — not a similarity scope.
-    # include_hidden is a gallery-browse flag; similar() has its OWN presentation
-    # exclusion (a similarity-quality concern, #1274), so drop it here (#141).
-    scope = {
-        k: v for k, v in filters.items() if k not in ("post_id", "include_hidden")
-    }
-    async with get_session() as session:
-        svc = GalleryService(session)
-        try:
-            images = await svc.similar(image_id=similar_to, limit=limit, **scope)
-        except ValueError as exc:
-            return jsonify({"error": str(exc)}), 400
-    if images is None:
-        return jsonify({"error": "not found"}), 404
-    return jsonify(
-        {
-            "images": [_image_json(i) for i in images],
-            "next_cursor": None,
-            "date_groups": [],
-        }
-    )
-
-
 @gallery_bp.route("/timeline", methods=["GET"])
 async def timeline():
-    try:
-        filters, _sort = _parse_filters()
-    except ValueError:
-        return jsonify({"error": "invalid filter parameter"}), 400
+    tag_id_raw = request.args.get("tag_id")
+    tag_id = int(tag_id_raw) if tag_id_raw else None
+    post_id_raw = request.args.get("post_id")
+    post_id = int(post_id_raw) if post_id_raw else None
+    artist_id_raw = request.args.get("artist_id")
+    artist_id = int(artist_id_raw) if artist_id_raw else None
     async with get_session() as session:
         svc = GalleryService(session)
         try:
-            buckets = await svc.timeline(**filters)
+            buckets = await svc.timeline(
+                tag_id=tag_id, post_id=post_id, artist_id=artist_id
+            )
         except ValueError as exc:
             return jsonify({"error": str(exc)}), 400
     return jsonify(
@@ -185,140 +78,31 @@ async def timeline():
     )
 
 
-@gallery_bp.route("/facets", methods=["GET"])
-async def facets():
-    try:
-        filters, _sort = _parse_filters()
-    except ValueError:
-        return jsonify({"error": "invalid filter parameter"}), 400
-    async with get_session() as session:
-        svc = GalleryService(session)
-        try:
-            f = await svc.facets(**filters)
-        except ValueError as exc:
-            return jsonify({"error": str(exc)}), 400
-    return jsonify(
-        {
-            "total": f.total,
-            "platforms": f.platforms,
-            "untagged": f.untagged,
-            "no_artist": f.no_artist,
-            "date_min": f.date_min.isoformat() if f.date_min else None,
-            "date_max": f.date_max.isoformat() if f.date_max else None,
-        }
-    )
-
-
 @gallery_bp.route("/jump", methods=["GET"])
 async def jump():
     try:
         year = int(request.args["year"])
         month = int(request.args["month"])
-        filters, sort = _parse_filters()
     except (KeyError, ValueError):
         return jsonify({"error": "year and month query params required"}), 400
+    tag_id_raw = request.args.get("tag_id")
+    tag_id = int(tag_id_raw) if tag_id_raw else None
+    post_id_raw = request.args.get("post_id")
+    post_id = int(post_id_raw) if post_id_raw else None
+    artist_id_raw = request.args.get("artist_id")
+    artist_id = int(artist_id_raw) if artist_id_raw else None
     async with get_session() as session:
         svc = GalleryService(session)
         try:
             cursor = await svc.jump_cursor(
-                year=year, month=month, sort=sort, **filters,
+                year=year, month=month, tag_id=tag_id,
+                post_id=post_id, artist_id=artist_id,
             )
         except ValueError as exc:
             return jsonify({"error": str(exc)}), 400
     return jsonify({"cursor": cursor})
 
 
-# -- Hidden-view review (#141): auto-hidden chrome flagged "also looks like
-#    content", surfaced in the gallery's Show-hidden review strip. -----------
-@gallery_bp.route("/hidden-review", methods=["GET"])
-async def hidden_review():
-    """Unresolved presentation auto-hide flags, most-concerning first (highest
-    content score) — for the gallery's Hidden-view review strip."""
-    ptag = aliased(Tag)
-    ctag = aliased(Tag)
-    async with get_session() as session:
-        rows = (await session.execute(
-            select(
-                PresentationReview.image_record_id,
-                PresentationReview.tag_id,
-                PresentationReview.conflict_tag_id,
-                PresentationReview.conflict_score,
-                ImageRecord.path, ImageRecord.thumbnail_path,
-                ImageRecord.sha256, ImageRecord.mime,
-                ptag.name.label("tag_name"),
-                ctag.name.label("conflict_name"),
-            )
-            .join(ImageRecord, ImageRecord.id == PresentationReview.image_record_id)
-            .join(ptag, ptag.id == PresentationReview.tag_id)
-            .outerjoin(ctag, ctag.id == PresentationReview.conflict_tag_id)
-            .where(PresentationReview.resolved_at.is_(None))
-            .order_by(PresentationReview.conflict_score.desc())
-        )).all()
-    return jsonify({"items": [
-        {
-            "image_id": r.image_record_id,
-            "tag_id": r.tag_id,
-            "tag_name": r.tag_name,
-            "conflict_tag_id": r.conflict_tag_id,
-            "conflict_name": r.conflict_name,
-            "conflict_score": r.conflict_score,
-            "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
-            "image_url": image_url(r.path),
-        }
-        for r in rows
-    ]})
-
-
-@gallery_bp.route(
-    "/hidden-review/<int:image_id>/<int:tag_id>/keep", methods=["POST"]
-)
-async def hidden_review_keep(image_id, tag_id):
-    """Keep the auto-hide: resolve the flag; the tag stays applied (#141)."""
-    async with get_session() as session:
-        await session.execute(
-            update(PresentationReview)
-            .where(
-                PresentationReview.image_record_id == image_id,
-                PresentationReview.tag_id == tag_id,
-            )
-            .values(resolved_at=datetime.now(UTC))
-        )
-        await session.commit()
-    return "", 204
-
-
-@gallery_bp.route(
-    "/hidden-review/<int:image_id>/<int:tag_id>/unhide", methods=["POST"]
-)
-async def hidden_review_unhide(image_id, tag_id):
-    """Un-hide: remove the presentation tag (image returns to the gallery), record
-    a rejection so the head LEARNS it misfired, and resolve the flag (#141)."""
-    from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-    async with get_session() as session:
-        await session.execute(
-            delete(image_tag).where(
-                image_tag.c.image_record_id == image_id,
-                image_tag.c.tag_id == tag_id,
-            )
-        )
-        await session.execute(
-            pg_insert(TagSuggestionRejection)
-            .values(image_record_id=image_id, tag_id=tag_id)
-            .on_conflict_do_nothing()
-        )
-        await session.execute(
-            update(PresentationReview)
-            .where(
-                PresentationReview.image_record_id == image_id,
-                PresentationReview.tag_id == tag_id,
-            )
-            .values(resolved_at=datetime.now(UTC))
-        )
-        await session.commit()
-    return "", 204
-
-
 @gallery_bp.route("/image/<int:image_id>", methods=["GET"])
 async def image_detail(image_id: int):
     async with get_session() as session:

backend/app/api/gpu.py

@@ -1,422 +0,0 @@
-"""GPU-job API (#114): the HTTP surface the desktop agent pulls work from.
-
-The agent stays HTTP-only — it leases jobs, fetches image pixels via the normal
-FC image URLs, and submits embeddings/regions back, all over this API. Redis and
-Postgres are never exposed. The agent endpoints are gated by a bearer token
-(Authorization: Bearer <token>) stored in AppSetting; the admin endpoints
-(token / backfill / status) ride the browser session like the rest of FC's
-homelab admin.
-"""
-
-import secrets
-from pathlib import Path
-
-from quart import Blueprint, jsonify, request
-from sqlalchemy import func, or_, select, update
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from ..extensions import get_session
-from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
-from ..services.gallery_service import image_url
-from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
-from ..services.ml.gpu_triage import classify_reason, recover_defective_image
-from ..services.ml.regions import RegionService
-
-gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
-
-# Same container mount the maintenance tasks use (tasks/admin.py) — recovery
-# deletes the defective original + thumbnail under it.
-_IMAGES_ROOT = Path("/images")
-
-_TOKEN_KEY = "gpu_agent_token"
-
-
-def _bearer() -> str | None:
-    h = request.headers.get("Authorization", "")
-    return h[7:].strip() if h.startswith("Bearer ") else None
-
-
-async def _agent_authed(session) -> bool:
-    supplied = _bearer()
-    if not supplied:
-        return False
-    stored = (
-        await session.execute(
-            select(AppSetting.value).where(AppSetting.key == _TOKEN_KEY)
-        )
-    ).scalar_one_or_none()
-    return stored is not None and secrets.compare_digest(supplied, stored)
-
-
-# --- Admin (browser): token + backfill + status -------------------------
-
-@gpu_bp.route("/token", methods=["GET"])
-async def get_token():
-    async with get_session() as session:
-        tok = (
-            await session.execute(
-                select(AppSetting.value).where(AppSetting.key == _TOKEN_KEY)
-            )
-        ).scalar_one_or_none()
-    return jsonify({"token": tok, "configured": tok is not None})
-
-
-@gpu_bp.route("/token/rotate", methods=["POST"])
-async def rotate_token():
-    token = secrets.token_urlsafe(32)
-    async with get_session() as session:
-        await session.execute(
-            pg_insert(AppSetting)
-            .values(key=_TOKEN_KEY, value=token)
-            .on_conflict_do_update(index_elements=["key"], set_={"value": token})
-        )
-        await session.commit()
-    return jsonify({"token": token})
-
-
-@gpu_bp.route("/status", methods=["GET"])
-async def status():
-    async with get_session() as session:
-        rows = (
-            await session.execute(
-                select(GpuJob.status, func.count()).group_by(GpuJob.status)
-            )
-        ).all()
-    counts = dict(rows)
-    return jsonify({
-        "pending": counts.get("pending", 0),
-        "leased": counts.get("leased", 0),
-        "done": counts.get("done", 0),
-        "error": counts.get("error", 0),
-    })
-
-
-@gpu_bp.route("/backfill", methods=["POST"])
-async def backfill():
-    """Enqueue a job for every image that doesn't already have one for `task`."""
-    body = await request.get_json(silent=True) or {}
-    task = str(body.get("task") or "ccip")
-    from ..tasks.gpu_queue import enqueue_gpu_backfill
-
-    r = enqueue_gpu_backfill.delay(task)
-    return jsonify({"celery_task_id": r.id, "task": task}), 202
-
-
-@gpu_bp.route("/reprocess", methods=["POST"])
-async def reprocess():
-    """Reset every done/error job of `task` back to pending so the agent re-runs
-    the WHOLE library under the current pipeline (e.g. after adding crop
-    detectors). Heavy — the back-catalogue is otherwise skipped by the backfills."""
-    body = await request.get_json(silent=True) or {}
-    task = str(body.get("task") or "ccip")
-    from ..tasks.gpu_queue import reprocess_gpu_jobs
-
-    r = reprocess_gpu_jobs.delay(task)
-    return jsonify({"celery_task_id": r.id, "task": task}), 202
-
-
-@gpu_bp.route("/retry_errors", methods=["POST"])
-async def retry_errors():
-    """Requeue every ERRORED job (all task types) back to pending — the scoped
-    recovery after an agent-side fix (e.g. the short-video sampler), where
-    /reprocess would needlessly re-run the whole done library too. Attempts and
-    the stored error reset so each job gets its full retry budget under the
-    fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
-    rows a later success made moot — the same statements the backfills run), so
-    one failing file requeues as ONE job, never a fan-out of duplicates. Small
-    row count (errors only) → inline statements; the response carries the
-    counts for the UI toast. Triage-confirmed defects are NOT requeued (see
-    the WHERE below) — they stay on the recovery surface."""
-    async with get_session() as session:
-        pruned = 0
-        for stmt in error_dedupe_statements():
-            pruned += (await session.execute(stmt)).rowcount or 0
-        res = await session.execute(
-            update(GpuJob)
-            .where(
-                GpuJob.status == "error",
-                # Triage-confirmed DEFECTS stay errored: the integrity probe
-                # already proved the FILE is bad, so re-running the job just
-                # burns agent time re-minting the same tombstone — those go
-                # through /errors/<id>/recover instead.
-                or_(GpuJob.triage_status.is_(None),
-                    GpuJob.triage_status != "defect"),
-            )
-            .values(
-                status="pending", attempts=0, error=None, lease_token=None,
-                leased_at=None, lease_expires_at=None, triage_status=None,
-                updated_at=func.now(),
-            )
-        )
-        kept = (
-            await session.execute(
-                select(func.count()).select_from(GpuJob)
-                .where(GpuJob.status == "error")
-            )
-        ).scalar_one()
-        await session.commit()
-    return jsonify({
-        "requeued": res.rowcount or 0, "pruned": pruned, "defects_kept": kept,
-    })
-
-
-# --- Failure triage + recovery (#125) ------------------------------------
-
-@gpu_bp.route("/errors", methods=["GET"])
-async def errors():
-    """The triage view of the error tombstones: every errored job joined with
-    its image's integrity verdict, bucketed by reason for the overview. The
-    probe sweep (triage_gpu_errors, 15-min beat) fills triage_status; 'defect'
-    rows are the recovery surface's list."""
-    async with get_session() as session:
-        rows = (
-            await session.execute(
-                select(
-                    GpuJob.id, GpuJob.image_record_id, GpuJob.task,
-                    GpuJob.error, GpuJob.triage_status, GpuJob.updated_at,
-                    ImageRecord.integrity_status, ImageRecord.mime,
-                    ImageRecord.path, ImageRecord.thumbnail_path,
-                )
-                .join(ImageRecord, ImageRecord.id == GpuJob.image_record_id)
-                .where(GpuJob.status == "error")
-                .order_by(GpuJob.updated_at.desc())
-                .limit(500)
-            )
-        ).all()
-        total = (
-            await session.execute(
-                select(func.count()).select_from(GpuJob)
-                .where(GpuJob.status == "error")
-            )
-        ).scalar_one()
-    by_class: dict[str, int] = {}
-    triage = {"defect": 0, "file_ok": 0, "unclassified": 0}
-    items = []
-    for r in rows:
-        cls = classify_reason(r.error)
-        by_class[cls] = by_class.get(cls, 0) + 1
-        bucket = r.triage_status or "unclassified"
-        triage[bucket] = triage.get(bucket, 0) + 1
-        items.append({
-            "job_id": r.id,
-            "image_id": r.image_record_id,
-            "task": r.task,
-            "error": r.error,
-            "reason_class": cls,
-            "triage_status": r.triage_status,
-            "integrity_status": r.integrity_status,
-            "mime": r.mime,
-            "image_url": image_url(r.path),
-            "thumbnail_url": (
-                image_url(r.thumbnail_path) if r.thumbnail_path else None
-            ),
-            "updated_at": r.updated_at.isoformat() if r.updated_at else None,
-        })
-    return jsonify({
-        "total": total, "by_class": by_class, "triage": triage, "items": items,
-    })
-
-
-@gpu_bp.route("/errors/triage", methods=["POST"])
-async def errors_triage():
-    """Run the probe sweep NOW (the card's button) instead of waiting out the
-    15-minute beat cadence."""
-    from ..tasks.maintenance import triage_gpu_errors
-
-    r = triage_gpu_errors.delay()
-    return jsonify({"celery_task_id": r.id}), 202
-
-
-@gpu_bp.route("/errors/<int:image_id>/recover", methods=["POST"])
-async def errors_recover(image_id: int):
-    """Recover a defect-triaged original: delete the bad copy + record and
-    re-poll its subscription Source (a fresh fetch re-imports the file, which
-    re-enters the GPU pipeline). Returns status 'no_source' when nothing
-    pollable resolves — the file needs manual replacement there."""
-    async with get_session() as session:
-        result = await session.run_sync(
-            lambda s: recover_defective_image(
-                s, image_id, images_root=_IMAGES_ROOT,
-            )
-        )
-    return jsonify(result)
-
-
-# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
-
-@gpu_bp.route("/jobs/lease", methods=["POST"])
-async def lease():
-    body = await request.get_json(silent=True) or {}
-    agent_id = str(body.get("agent_id") or "agent")
-    try:
-        batch = min(max(int(body.get("batch_size", 8)), 1), 64)
-    except (TypeError, ValueError):
-        batch = 8
-    async with get_session() as session:
-        if not await _agent_authed(session):
-            return jsonify({"error": "unauthorized"}), 401
-        jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
-        ml = (
-            await session.execute(select(MLSettings).where(MLSettings.id == 1))
-        ).scalar_one()
-        # image rows for url/mime in one shot
-        ids = [j.image_record_id for j in jobs]
-        imgs = {
-            i.id: i for i in (
-                await session.execute(
-                    select(ImageRecord).where(ImageRecord.id.in_(ids))
-                )
-            ).scalars()
-        } if ids else {}
-        await session.commit()
-        # Crop-proposer config, announced FROM THE SETTING like embed_model_name
-        # (#134): the agent builds its detectors from this, rebuilding live when
-        # it changes — so tuning is a DB/UI edit, never an agent restart. Same
-        # block for every job in the batch (it's global), built once. An enabled
-        # toggle off is carried through so the agent skips that proposer.
-        detectors = {
-            "person": {
-                "enabled": ml.detector_person_enabled,
-                "weights": ml.detector_person_weights,
-                "conf": ml.detector_person_conf,
-            },
-            "anatomy": {
-                "enabled": ml.detector_anatomy_enabled,
-                "weights": ml.detector_anatomy_weights,
-                "conf": ml.detector_anatomy_conf,
-            },
-            "panel": {
-                "enabled": ml.detector_panel_enabled,
-                "weights": ml.detector_panel_weights,
-                "conf": ml.detector_panel_conf,
-            },
-            "max_figures": ml.detector_max_figures,
-            "max_components": ml.detector_max_components,
-            "max_panels": ml.detector_max_panels,
-            "max_regions": ml.detector_max_regions,
-            "dedupe_iou": ml.detector_dedupe_iou,
-        }
-        out = []
-        for j in jobs:
-            img = imgs.get(j.image_record_id)
-            if img is None:
-                continue
-            out.append({
-                "job_id": j.id,
-                "image_id": j.image_record_id,
-                "task": j.task,
-                "mime": img.mime,
-                "image_url": image_url(img.path),
-                # For video/animated: the agent samples at this cadence.
-                "frame_interval_seconds": ml.video_frame_interval_seconds,
-                "max_frames": ml.video_max_frames,
-                # The embedding model the agent must use for concept crops + the
-                # whole-image 'embed' task, so its vectors land in the SAME space
-                # the heads trained in. Server-announced FROM THE SETTING → the
-                # agent stays model-agnostic; an operator swap is a setting + a
-                # re-embed, never an agent change.
-                "embed_model_name": ml.embedder_model_name,
-                "embed_version": ml.embedder_model_version,
-                "detectors": detectors,
-            })
-    return jsonify({"jobs": out})
-
-
-@gpu_bp.route("/jobs/heartbeat", methods=["POST"])
-async def heartbeat():
-    body = await request.get_json(silent=True) or {}
-    agent_id = str(body.get("agent_id") or "agent")
-    job_ids = [int(x) for x in (body.get("job_ids") or [])]
-    async with get_session() as session:
-        if not await _agent_authed(session):
-            return jsonify({"error": "unauthorized"}), 401
-        n = await GpuJobService(session).heartbeat(agent_id, job_ids)
-        await session.commit()
-    return jsonify({"extended": n})
-
-
-@gpu_bp.route("/jobs/submit", methods=["POST"])
-async def submit():
-    """Store a job's regions + close it. regions: [{kind, bbox:[x,y,w,h],
-    frame_time?, score?, *_version?, ccip_embedding?, siglip_embedding?}].
-    replace_kinds defaults to the kinds present in the submitted regions."""
-    body = await request.get_json(silent=True) or {}
-    agent_id = str(body.get("agent_id") or "agent")
-    job_id = body.get("job_id")
-    regions = body.get("regions") or []
-    if job_id is None:
-        return jsonify({"error": "job_id required"}), 400
-    kinds = body.get("replace_kinds") or sorted({r["kind"] for r in regions})
-    async with get_session() as session:
-        if not await _agent_authed(session):
-            return jsonify({"error": "unauthorized"}), 401
-        job = await session.get(GpuJob, int(job_id))
-        if job is None or job.status != "leased" or job.lease_token != agent_id:
-            return jsonify({"error": "lease_invalid"}), 409
-        if kinds:
-            await RegionService(session).replace_regions(
-                job.image_record_id, kinds, regions
-            )
-        await GpuJobService(session).complete(agent_id, int(job_id))
-        await session.commit()
-    return jsonify({"ok": True, "stored": len(regions)})
-
-
-@gpu_bp.route("/jobs/submit_embedding", methods=["POST"])
-async def submit_embedding():
-    """Store a whole-image SigLIP embedding (the 'embed' task) on image_record +
-    close the job. Body: {agent_id, job_id, embedding:[...], embedding_version}.
-    This is how the GPU agent re-embeds the library under a new model (#1190) —
-    much faster than the CPU ml-worker at higher resolutions."""
-    body = await request.get_json(silent=True) or {}
-    agent_id = str(body.get("agent_id") or "agent")
-    job_id = body.get("job_id")
-    embedding = body.get("embedding")
-    version = body.get("embedding_version")
-    if job_id is None or not embedding or not version:
-        return jsonify({"error": "job_id, embedding, embedding_version required"}), 400
-    async with get_session() as session:
-        if not await _agent_authed(session):
-            return jsonify({"error": "unauthorized"}), 401
-        job = await session.get(GpuJob, int(job_id))
-        if job is None or job.status != "leased" or job.lease_token != agent_id:
-            return jsonify({"error": "lease_invalid"}), 409
-        img = await session.get(ImageRecord, job.image_record_id)
-        if img is not None:
-            img.siglip_embedding = embedding
-            img.siglip_model_version = version
-        await GpuJobService(session).complete(agent_id, int(job_id))
-        await session.commit()
-    return jsonify({"ok": True})
-
-
-@gpu_bp.route("/jobs/fail", methods=["POST"])
-async def fail():
-    body = await request.get_json(silent=True) or {}
-    agent_id = str(body.get("agent_id") or "agent")
-    job_id = body.get("job_id")
-    if job_id is None:
-        return jsonify({"error": "job_id required"}), 400
-    async with get_session() as session:
-        if not await _agent_authed(session):
-            return jsonify({"error": "unauthorized"}), 401
-        ok = await GpuJobService(session).fail(
-            agent_id, int(job_id), str(body.get("error") or "")
-        )
-        await session.commit()
-    return jsonify({"ok": ok})
-
-
-@gpu_bp.route("/jobs/release", methods=["POST"])
-async def release():
-    """Graceful stop: the agent hands its still-leased jobs back to pending so
-    they're picked up immediately instead of waiting out the lease."""
-    body = await request.get_json(silent=True) or {}
-    agent_id = str(body.get("agent_id") or "agent")
-    job_ids = [int(x) for x in (body.get("job_ids") or [])]
-    async with get_session() as session:
-        if not await _agent_authed(session):
-            return jsonify({"error": "unauthorized"}), 401
-        n = await GpuJobService(session).release(agent_id, job_ids)
-        await session.commit()
-    return jsonify({"released": n})

backend/app/api/heads.py

@@ -1,285 +0,0 @@
-"""Heads API (#114): train + inspect the per-concept heads that power
-suggestions (replacing Camie + centroid).
-
-POST /api/heads/train   — (re)train all eligible heads (one run at a time).
-GET  /api/heads         — status: head count, last-trained, running run, the
-                          per-concept head table (strength + auto-apply ready),
-                          and recent training runs. The card rehydrates from
-                          here so status survives navigation.
-"""
-
-from quart import Blueprint, jsonify, request
-from sqlalchemy import desc, func, select
-
-from ..extensions import get_session
-from ..models import (
-    HeadAutoApplyRun,
-    HeadMetric,
-    HeadMetricsSnapshot,
-    HeadTrainingRun,
-    Tag,
-    TagHead,
-)
-from ..models.tag import image_tag
-from ..services.ml.heads import (
-    HeadAutoApplyAlreadyRunning,
-    HeadAutoApplyDisabled,
-    HeadTrainingAlreadyRunning,
-    start_head_auto_apply_run,
-    start_head_training_run,
-)
-
-heads_bp = Blueprint("heads", __name__, url_prefix="/api/heads")
-
-
-def _serialize_run(run: HeadTrainingRun) -> dict:
-    return {
-        "id": run.id,
-        "params": run.params,
-        "status": run.status,
-        "started_at": run.started_at.isoformat() if run.started_at else None,
-        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
-        "n_trained": run.n_trained,
-        "n_skipped": run.n_skipped,
-        "error": run.error,
-    }
-
-
-@heads_bp.route("/train", methods=["POST"])
-async def train():
-    body = await request.get_json(silent=True) or {}
-    params = body.get("params") or body or {}
-    async with get_session() as session:
-        try:
-            run_id = await session.run_sync(
-                lambda s: start_head_training_run(s, params)
-            )
-        except HeadTrainingAlreadyRunning as running:
-            return jsonify({
-                "error": "training_already_running",
-                "running_id": int(running.args[0]),
-            }), 409
-        await session.commit()
-    return jsonify({"run_id": run_id, "status": "running"}), 202
-
-
-@heads_bp.route("", methods=["GET"])
-async def status():
-    async with get_session() as session:
-        count, last_trained = (
-            await session.execute(
-                select(func.count(), func.max(TagHead.trained_at))
-            )
-        ).one()
-        graduated = (
-            await session.execute(
-                select(func.count()).where(
-                    TagHead.auto_apply_threshold.is_not(None)
-                )
-            )
-        ).scalar_one()
-        running = (
-            await session.execute(
-                select(HeadTrainingRun.id)
-                .where(HeadTrainingRun.status == "running")
-                .order_by(HeadTrainingRun.id.desc())
-                .limit(1)
-            )
-        ).scalar_one_or_none()
-        runs = (
-            await session.execute(
-                select(HeadTrainingRun)
-                .order_by(HeadTrainingRun.id.desc())
-                .limit(10)
-            )
-        ).scalars().all()
-        # The per-concept table: strongest first, capped for the admin card.
-        head_rows = (
-            await session.execute(
-                select(
-                    TagHead.tag_id, Tag.name, Tag.kind,
-                    TagHead.n_pos, TagHead.n_neg, TagHead.ap,
-                    TagHead.precision_cv, TagHead.recall,
-                    TagHead.auto_apply_threshold, TagHead.trained_at,
-                )
-                .join(Tag, Tag.id == TagHead.tag_id)
-                .order_by(desc(TagHead.ap))
-                .limit(500)
-            )
-        ).all()
-        heads = [
-            {
-                "tag_id": r.tag_id,
-                "name": r.name,
-                "category": r.kind.value if hasattr(r.kind, "value") else str(r.kind),
-                "n_pos": r.n_pos,
-                "n_neg": r.n_neg,
-                "ap": r.ap,
-                "precision": r.precision_cv,
-                "recall": r.recall,
-                "auto_apply": r.auto_apply_threshold is not None,
-                "trained_at": r.trained_at.isoformat() if r.trained_at else None,
-            }
-            for r in head_rows
-        ]
-    return jsonify({
-        "head_count": count,
-        "graduated_count": graduated,
-        "last_trained_at": last_trained.isoformat() if last_trained else None,
-        "running_id": running,
-        "runs": [_serialize_run(r) for r in runs],
-        "heads": heads,
-    })
-
-
-def _serialize_apply_run(run: HeadAutoApplyRun) -> dict:
-    return {
-        "id": run.id,
-        "dry_run": run.dry_run,
-        "status": run.status,
-        "started_at": run.started_at.isoformat() if run.started_at else None,
-        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
-        "n_applied": run.n_applied,
-        "report": run.report,
-        "error": run.error,
-    }
-
-
-@heads_bp.route("/auto-apply", methods=["POST"])
-async def auto_apply():
-    """Trigger an earned-auto-apply sweep. {dry_run:true} previews (writes
-    nothing); a real sweep needs head_auto_apply_enabled on."""
-    body = await request.get_json(silent=True) or {}
-    params = {"dry_run": bool(body.get("dry_run", False))}
-    async with get_session() as session:
-        try:
-            run_id = await session.run_sync(
-                lambda s: start_head_auto_apply_run(s, params)
-            )
-        except HeadAutoApplyAlreadyRunning as running:
-            return jsonify({
-                "error": "auto_apply_already_running",
-                "running_id": int(running.args[0]),
-            }), 409
-        except HeadAutoApplyDisabled:
-            return jsonify({"error": "auto_apply_disabled"}), 400
-        await session.commit()
-    return jsonify({"run_id": run_id, "status": "running"}), 202
-
-
-@heads_bp.route("/auto-apply", methods=["GET"])
-async def auto_apply_status():
-    async with get_session() as session:
-        running = (
-            await session.execute(
-                select(HeadAutoApplyRun.id)
-                .where(HeadAutoApplyRun.status == "running")
-                .order_by(HeadAutoApplyRun.id.desc())
-                .limit(1)
-            )
-        ).scalar_one_or_none()
-        runs = (
-            await session.execute(
-                select(HeadAutoApplyRun)
-                .order_by(HeadAutoApplyRun.id.desc())
-                .limit(10)
-            )
-        ).scalars().all()
-    return jsonify({
-        "running_id": running,
-        "runs": [_serialize_apply_run(r) for r in runs],
-    })
-
-
-@heads_bp.route("/metrics", methods=["GET"])
-async def metrics():
-    """Auto-apply observability: per-concept current counts (volume, misfires,
-    under-fires, realized misfire rate, head quality) + the daily time-series so
-    the operator can tune the precision target + support floor from real data."""
-    async with get_session() as session:
-        head_rows = (
-            await session.execute(
-                select(
-                    TagHead.tag_id, Tag.name, TagHead.ap, TagHead.precision_cv,
-                    TagHead.recall, TagHead.auto_apply_threshold, TagHead.n_pos,
-                ).join(Tag, Tag.id == TagHead.tag_id)
-            )
-        ).all()
-        heads = {r.tag_id: r for r in head_rows}
-        metric_rows = (
-            await session.execute(
-                select(
-                    HeadMetric.tag_id, HeadMetric.n_misfires, HeadMetric.n_underfires
-                )
-            )
-        ).all()
-        mets = {r.tag_id: r for r in metric_rows}
-        applied = dict(
-            (
-                await session.execute(
-                    select(image_tag.c.tag_id, func.count())
-                    .where(image_tag.c.source == "head_auto")
-                    .group_by(image_tag.c.tag_id)
-                )
-            ).all()
-        )
-        names = {r.tag_id: r.name for r in head_rows}
-        # Names for metric-only tags (head pruned but corrections recorded).
-        missing = [t for t in mets if t not in names]
-        if missing:
-            for tid, nm in (
-                await session.execute(
-                    select(Tag.id, Tag.name).where(Tag.id.in_(missing))
-                )
-            ).all():
-                names[tid] = nm
-
-        concepts = []
-        for tid in set(heads) | set(mets):
-            h = heads.get(tid)
-            m = mets.get(tid)
-            n_applied = applied.get(tid, 0)
-            n_mis = m.n_misfires if m else 0
-            denom = n_applied + n_mis
-            concepts.append({
-                "tag_id": tid,
-                "name": names.get(tid, str(tid)),
-                "n_auto_applied": n_applied,
-                "n_misfires": n_mis,
-                "n_underfires": m.n_underfires if m else 0,
-                # Of everything this head ever auto-applied, the fraction you
-                # removed — the misfire rate (null until something fired).
-                "misfire_rate": round(n_mis / denom, 4) if denom else None,
-                "ap": h.ap if h else None,
-                "precision_cv": h.precision_cv if h else None,
-                "recall": h.recall if h else None,
-                "auto_apply": bool(h and h.auto_apply_threshold is not None),
-                "n_pos": h.n_pos if h else None,
-            })
-        concepts.sort(key=lambda c: (c["n_misfires"], c["n_auto_applied"]), reverse=True)
-
-        snaps = (
-            await session.execute(
-                select(HeadMetricsSnapshot)
-                .order_by(HeadMetricsSnapshot.snapshot_at.desc())
-                .limit(1000)
-            )
-        ).scalars().all()
-    return jsonify({
-        "concepts": concepts,
-        "snapshots": [
-            {
-                "tag_id": s.tag_id,
-                "name": s.name,
-                "snapshot_at": s.snapshot_at.isoformat() if s.snapshot_at else None,
-                "n_auto_applied": s.n_auto_applied,
-                "n_misfires": s.n_misfires,
-                "n_underfires": s.n_underfires,
-                "ap": s.ap,
-                "precision_cv": s.precision_cv,
-                "recall": s.recall,
-                "n_pos": s.n_pos,
-            }
-            for s in snaps
-        ],
-    })

backend/app/api/import_admin.py

@@ -35,26 +35,10 @@ async def trigger_scan():
 @import_admin_bp.route("/status", methods=["GET"])
 async def status():
     async with get_session() as session:
-        # Active batch = running batch that still has outstanding work.
-        # Plain "most recent running" picks freshly-created scans that
-        # enqueued zero new files and hides the older batch that's
-        # actually being processed. Mirrors the EXISTS predicate
-        # /api/system/stats already uses (api/settings.py:145-160).
-        # Audit 2026-06-02 — /api/import/status and /api/system/stats
-        # used to disagree on the active-batch predicate; the UI banner
-        # said "Scanning…" indefinitely while the stats card said idle.
         active = (
             await session.execute(
                 select(ImportBatch)
-                .where(
-                    ImportBatch.status == "running",
-                    select(ImportTask.id)
-                    .where(
-                        ImportTask.batch_id == ImportBatch.id,
-                        ImportTask.status.in_(["pending", "queued", "processing"]),
-                    )
-                    .exists(),
-                )
+                .where(ImportBatch.status == "running")
                 .order_by(ImportBatch.started_at.desc())
                 .limit(1)
             )
@@ -130,53 +114,18 @@ async def retry_failed():
                 status="queued", error=None,
                 started_at=None, finished_at=None,
             )
-            .returning(ImportTask.id, ImportTask.task_type)
+            .returning(ImportTask.id)
         )
-        failed = result.all()
-        if not failed:
+        failed_ids = [row[0] for row in result.all()]
+        if not failed_ids:
             return jsonify({"retried": 0})
         await session.commit()
 
-    from ..tasks.import_file import enqueue_import
-    for tid, task_type in failed:
-        enqueue_import(tid, task_type)
+    from ..tasks.import_file import import_media_file
+    for tid in failed_ids:
+        import_media_file.delay(tid)
 
-    return jsonify({"retried": len(failed)})
-
-
-@import_admin_bp.route("/tasks/<int:task_id>/refetch", methods=["POST"])
-async def refetch_task(task_id: int):
-    """Layer-2 one-shot re-download: delete the (corrupt) file behind a
-    failed import task and re-run its source's downloader to fetch a
-    fresh copy. Only works for files that resolve to an enabled,
-    real-URL subscription Source; filesystem-only imports return
-    no_source.
-
-    Returns one of: refetch_queued (+source_id) / no_source /
-    already_refetched / not_found / not_failed.
-    """
-    async with get_session() as session:
-        result = await session.run_sync(_refetch_task_sync, task_id)
-    if result["status"] == "not_found":
-        return jsonify(result), 404
-    if result["status"] == "not_failed":
-        return jsonify(result), 400
-    return jsonify(result)
-
-
-def _refetch_task_sync(session, task_id: int) -> dict:
-    from pathlib import Path
-
-    from ..models import ImportSettings
-    from ..services.refetch_service import attempt_refetch
-
-    task = session.get(ImportTask, task_id)
-    if task is None:
-        return {"status": "not_found"}
-    if task.status != "failed":
-        return {"status": "not_failed"}
-    settings = ImportSettings.load_sync(session)
-    return attempt_refetch(session, task, Path(settings.import_scan_path))
+    return jsonify({"retried": len(failed_ids)})
 
 
 @import_admin_bp.route("/clear-stuck", methods=["POST"])