Merge pull request 'fix(ext-ci): match AMO-renamed signed XPI' (#14) from dev into main

_apply_sidecar is additive: it find-or-creates Post/Source/ImageProvenance
and sets primary_post_id NULL-only, so any IR-migration provenance on the
existing row survives untouched and the new GS sidecar adds a second
ImageProvenance pointing at the freshly-created Post.

Wrapped in try/except so a malformed sidecar can't unwind the file-swap
commit — the file replacement is the critical operation.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.forgejo/workflows/ci.yml

@@ -24,11 +24,26 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Cache pip wheels
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
+
       - name: Install Python deps
         # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
         # Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
         # versions live on the runner image, not here.
-        run: pip install -r requirements.txt pytest pytest-asyncio
+        # uv: 5-10x faster wheel resolve than pip for cold caches.
+        # Falls back to pip install on uv-missing runners (older images).
+        run: |
+          if command -v uv >/dev/null 2>&1; then
+            uv pip install --system -r requirements.txt pytest pytest-asyncio
+          else
+            pip install -r requirements.txt pytest pytest-asyncio
+          fi
 
       - name: Ruff lint
         run: ruff check backend/ tests/ alembic/
@@ -99,6 +114,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Cache pip wheels
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
+
       - name: Integration suite (resolve service IPs, migrate, test)
         run: |
           set -eux
@@ -119,6 +142,11 @@ jobs:
             (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
             sleep 2
           done
-          pip install -r requirements.txt pytest pytest-asyncio
+          # uv when available (5-10x faster wheel resolve); fall back to pip.
+          if command -v uv >/dev/null 2>&1; then
+            uv pip install --system -r requirements.txt pytest pytest-asyncio
+          else
+            pip install -r requirements.txt pytest pytest-asyncio
+          fi
           alembic upgrade head
-          pytest tests/ -v -m integration
+          pytest tests/ -v -m integration --durations=25

.forgejo/workflows/extension.yml

@@ -39,14 +39,26 @@ jobs:
       - name: Commit signed XPI to frontend/public/extension/
         run: |
           set -e
-          XPI=$(ls extension/web-ext-artifacts/fabledcurator-*.xpi | head -1)
+          # AMO renames signed XPIs using its internal addon-id-safe-string
+          # (e.g. 997017ca3e104e30a75a-1.0.1.xpi), NOT our gecko ID. The
+          # original 'fabledcurator-*' glob never matched and the step
+          # exited 1 even on a successful sign (operator-flagged
+          # 2026-05-25). Match any *.xpi in the artifacts dir — fresh
+          # CI runner means there's exactly one — and rename it on copy
+          # so the FC server's whitelist (backend/app/frontend.py expects
+          # 'fabledcurator-*.xpi') keeps working.
+          XPI=$(ls extension/web-ext-artifacts/*.xpi 2>/dev/null | head -1)
           if [ -z "$XPI" ]; then
             echo "No XPI produced by web-ext sign — exiting"
+            ls -la extension/web-ext-artifacts/ || true
             exit 1
           fi
+          VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
+          DEST="frontend/public/extension/fabledcurator-${VERSION}.xpi"
           mkdir -p frontend/public/extension
-          cp "$XPI" frontend/public/extension/
-          # Also copy as -latest.xpi so the FC server can serve a stable URL.
+          # Wipe any prior versions so the directory doesn't grow each release.
+          rm -f frontend/public/extension/fabledcurator-*.xpi
+          cp "$XPI" "$DEST"
           cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
           git config user.name  "FC extension CI"
           git config user.email "noreply@fabledsword.com"
@@ -54,6 +66,6 @@ jobs:
           if git diff --cached --quiet; then
             echo "No changes to commit"
           else
-            git commit -m "ext: publish signed XPI $(basename $XPI)"
+            git commit -m "ext: publish signed XPI fabledcurator-${VERSION}.xpi"
             git push origin HEAD:main
           fi

backend/app/api/import_admin.py

@@ -47,6 +47,8 @@ async def status():
         if active:
             payload["active_batch"] = {
                 "id": active.id,
+                "source_path": active.source_path,
+                "scan_mode": active.scan_mode,
                 "total_files": active.total_files,
                 "imported": active.imported,
                 "skipped": active.skipped,

backend/app/services/importer.py

@@ -722,6 +722,15 @@ class Importer:
         row id (so tags/series/curation stay attached). ML is cleared so
         the import task re-derives it on the new pixels.
 
+        After the file swap, the new file's adjacent gallery-dl sidecar
+        (if any) is applied via _apply_sidecar — operator-flagged
+        2026-05-25: scanning a GS download dir with smaller IR-migrated
+        images on the receiving end used to swap files but lose the GS
+        sidecar's post metadata entirely. _apply_sidecar is additive
+        (find-or-create Post / Source / ImageProvenance, NULL-only
+        primary_post_id update) so any pre-existing Post linkage
+        survives untouched.
+
         If `new_path` is provided, `source` is assumed to ALREADY be at
         that path (FC-3c attach_in_place case) — skip the copy step.
         Otherwise the file is copied via _copy_to_library."""
@@ -751,6 +760,22 @@ class Importer:
         self.session.flush()
         self.session.commit()
 
+        # Sidecar enrichment from the new (larger) file's location.
+        # _apply_sidecar resolves artist from the sidecar itself if the
+        # existing row has none, and is internally guarded against
+        # missing-or-malformed sidecars (silent return).
+        try:
+            self._apply_sidecar(existing, source, None)
+        except Exception as exc:
+            # Don't unwind the supersede DB swap if sidecar parsing
+            # blows up unexpectedly — the file replacement is the
+            # critical operation, sidecar is enrichment.
+            log.warning(
+                "sidecar enrichment failed during supersede of "
+                "image_record.id=%s from %s: %s",
+                existing.id, source, exc,
+            )
+
         for stale in (old_path, old_thumb):
             if not stale or stale == str(dest):
                 # If the supersede kept the file in place (new_path == old

backend/app/services/ml/embedder.py

@@ -34,10 +34,21 @@ class Embedder:
         if self._model is not None:
             return
         import torch
-        from transformers import AutoModel, AutoProcessor
+        from transformers import AutoModel, SiglipImageProcessor
 
         self._torch = torch
-        self._processor = AutoProcessor.from_pretrained(str(self._model_dir))
+        # FC's embedder only does IMAGE inference — never text. AutoProcessor
+        # loads the full processor including SiglipTokenizer, which requires
+        # the sentencepiece library at import time even if we never call it.
+        # SiglipImageProcessor loads ONLY preprocessor_config.json (image
+        # side) and skips the tokenizer config entirely. Operator hit the
+        # ImportError 2026-05-25 once the ml-worker started actually running
+        # tag_and_embed; switching to the image-only loader avoids the
+        # tokenizer dep without adding ~30 MB of unused C++ build to the
+        # lean ml-worker image.
+        self._processor = SiglipImageProcessor.from_pretrained(
+            str(self._model_dir)
+        )
         self._model = AutoModel.from_pretrained(str(self._model_dir))
         self._model.eval()
 

extension/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "FabledCurator",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
 
   "browser_specific_settings": {

extension/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fabledcurator-extension",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "private": true,
   "description": "Firefox extension for FabledCurator",
   "scripts": {

frontend/src/components/settings/ImportTriggerPanel.vue

@@ -7,7 +7,8 @@
           indeterminate color="accent" size="20"
         />
         <span>
-          Scanning {{ store.activeBatch.source_path }} —
+          {{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
+          {{ store.activeBatch.source_path || '/import' }} —
           imported {{ store.activeBatch.imported }},
           skipped {{ store.activeBatch.skipped }},
           failed {{ store.activeBatch.failed }} /
@@ -24,8 +25,12 @@
 
       <p class="text-body-2 mb-3">
         <span v-if="!store.activeBatch">
-          Run a quick scan of the import directory. Deep scan (pHash dedup,
-          archives) lands in FC-2d.
+          <strong>Quick scan</strong> walks <code>/import</code> and enqueues
+          new files only (skips paths already on a non-failed ImportTask).
+          <strong>Deep scan</strong> additionally chains a phash backfill
+          across the existing library — use after bulk-imports to catch
+          near-duplicates that slipped through. Both modes route non-media
+          + sidecar pairs through PostAttachment capture.
         </span>
         <span v-else>
           An active batch is in progress. Wait for it to finish, or click
@@ -34,15 +39,26 @@
         </span>
       </p>
 
-      <v-btn
-        color="primary" rounded="pill"
-        :disabled="!!store.activeBatch"
-        :loading="busy"
-        @click="trigger"
-      >
-        <v-icon start>mdi-magnify-scan</v-icon>
-        Quick scan
-      </v-btn>
+      <div class="d-flex flex-wrap" style="gap: 12px;">
+        <v-btn
+          color="primary" rounded="pill"
+          :disabled="!!store.activeBatch"
+          :loading="busy === 'quick'"
+          @click="trigger('quick')"
+        >
+          <v-icon start>mdi-magnify-scan</v-icon>
+          Quick scan
+        </v-btn>
+        <v-btn
+          color="secondary" rounded="pill" variant="tonal"
+          :disabled="!!store.activeBatch"
+          :loading="busy === 'deep'"
+          @click="trigger('deep')"
+        >
+          <v-icon start>mdi-magnify-plus-outline</v-icon>
+          Deep scan
+        </v-btn>
+      </div>
 
       <v-alert v-if="store.triggerError" type="error" variant="tonal" class="mt-3" closable>
         {{ store.triggerError }}
@@ -56,12 +72,12 @@ import { ref } from 'vue'
 import { useImportStore } from '../../stores/import.js'
 
 const store = useImportStore()
-const busy = ref(false)
+const busy = ref(null)
 const clearing = ref(false)
 
-async function trigger() {
-  busy.value = true
-  try { await store.triggerScan() } catch {} finally { busy.value = false }
+async function trigger(mode) {
+  busy.value = mode
+  try { await store.triggerScan(mode) } catch {} finally { busy.value = null }
 }
 
 async function onClearStuck() {

frontend/src/stores/import.js

@@ -52,11 +52,37 @@ export const useImportStore = defineStore('import', () => {
     }
   }
 
-  async function triggerScan() {
+  async function triggerScan(mode = 'quick') {
+    if (!['quick', 'deep', 'verify'].includes(mode)) {
+      throw new Error(`unsupported scan mode: ${mode}`)
+    }
     triggerError.value = null
     try {
-      await api.post('/api/import/trigger', { body: { mode: 'quick' } })
+      await api.post('/api/import/trigger', { body: { mode } })
+      // Acknowledge immediately so the click isn't invisible. scan_directory
+      // can finalize the batch synchronously when every file in /import is
+      // already on a non-failed ImportTask (operator-flagged 2026-05-25:
+      // 233k existing tasks → all paths in skip-set → files_seen=0 →
+      // batch flashes 'running' for <100ms then 'complete' before the
+      // first refreshStatus() lands; UI never sees the active state).
+      const label = mode === 'deep'
+        ? 'Deep scan triggered (pHash backfill chained)'
+        : mode === 'verify' ? 'Library verify triggered' : 'Quick scan triggered'
+      window.__fcToast?.({ text: label, type: 'success' })
       await refreshStatus()
+      // Re-poll twice over ~5s to catch quick-finalize transitions and
+      // surface a result toast either way. Skip the "no new files" hint
+      // for 'verify' since it doesn't walk the import_root.
+      setTimeout(async () => {
+        await refreshStatus()
+        await loadTasks(true)
+        if (!activeBatch.value && mode !== 'verify') {
+          window.__fcToast?.({
+            text: 'Scan complete — no new files (everything already on an ImportTask row)',
+            type: 'info',
+          })
+        }
+      }, 2000)
     } catch (e) {
       triggerError.value = e.message
       window.__fcToast?.({ text: `Scan failed: ${e.message}`, type: 'error' })

frontend/src/views/SettingsView.vue

@@ -1,6 +1,16 @@
 <template>
   <v-container fluid class="py-6">
-    <v-tabs v-model="tab" color="accent" class="mb-4">
+    <!-- Sticky tabs: operator-flagged 2026-05-25 — long Import / Maintenance
+         panels pushed the tab strip out of the viewport, forcing a scroll-
+         to-top just to change tab. AppShell's TopNav is 64px sticky, so the
+         tab strip lives directly under it. Background uses the theme surface
+         token so it visually merges with the page rather than the
+         translucent v-tabs default. -->
+    <v-tabs
+      v-model="tab" color="accent" class="mb-4"
+      style="position: sticky; top: 64px; z-index: 4;
+             background: rgb(var(--v-theme-surface));"
+    >
       <v-tab value="overview">Overview</v-tab>
       <v-tab value="activity">Activity</v-tab>
       <v-tab value="import">Import</v-tab>
@@ -28,11 +38,14 @@
       </v-window-item>
 
       <v-window-item value="import">
+        <!-- Order: trigger → recent tasks → filters. Tasks sit directly
+             below the trigger so operator sees hit/miss feedback without
+             scrolling past the filter card (operator-flagged 2026-05-25). -->
         <ImportTriggerPanel />
         <v-divider class="my-6" />
-        <ImportFiltersForm />
-        <v-divider class="my-6" />
         <ImportTaskList />
+        <v-divider class="my-6" />
+        <ImportFiltersForm />
       </v-window-item>
 
       <v-window-item value="maintenance">

tests/test_phash_dedup.py

@@ -10,7 +10,15 @@ import pytest
 from PIL import Image
 from sqlalchemy import func, select
 
-from backend.app.models import ImageRecord, ImportSettings, Tag, TagKind
+from backend.app.models import (
+    ImageProvenance,
+    ImageRecord,
+    ImportSettings,
+    Post,
+    Source,
+    Tag,
+    TagKind,
+)
 from backend.app.models.tag import image_tag
 from backend.app.services.importer import Importer, SkipReason
 from backend.app.services.thumbnailer import Thumbnailer
@@ -141,6 +149,67 @@ def test_smaller_existing_is_superseded(importer, import_layout):
     assert Path(row.path).exists()
 
 
+def test_supersede_applies_new_file_sidecar(importer, import_layout):
+    """Operator-flagged 2026-05-25: scanning the GS download dir should
+    supersede smaller IR-migrated images AND wire up the GS sidecar's
+    Post/Source/ImageProvenance. Previously _supersede swapped the file
+    but ignored the sidecar entirely."""
+    import json
+    import_root, _ = import_layout
+
+    # Stage 1: a small, sidecar-less image (the "IR migration" precondition).
+    small = import_root / "ir-migration-folder" / "small.png"
+    _write(small, (60, 130, 200), (200, 200))
+    r1 = importer.import_one(small)
+    assert r1.status == "imported"
+    eid = r1.image_id
+
+    # Stage 2: a larger version of the same image (same phash) WITH a
+    # gallery-dl JSON sidecar adjacent. Live in a separate folder to
+    # simulate the GS download dir.
+    big = import_root / "Maewix" / "patreon" / "01_big.png"
+    _write(big, (60, 130, 200), (900, 900))
+    sidecar_path = big.with_suffix(big.suffix + ".json")
+    sidecar_path.parent.mkdir(parents=True, exist_ok=True)
+    sidecar_path.write_text(json.dumps({
+        "category": "patreon",
+        "id": 555,
+        "url": "https://www.patreon.com/posts/555",
+        "title": "Set 1",
+        "content": "<p>The big version</p>",
+        "page_count": 1,
+        "published_at": "2025-08-01T00:00:00+00:00",
+        "artist": "Maewix",
+    }))
+
+    r2 = importer.import_one(big)
+    assert r2.status == "superseded"
+    assert r2.image_id == eid
+
+    importer.session.expire_all()
+    # Row preserved, file replaced, sidecar metadata wired up.
+    row = importer.session.get(ImageRecord, eid)
+    assert row.width == 900 and row.height == 900
+
+    post = importer.session.execute(
+        select(Post).where(Post.external_post_id == "555")
+    ).scalar_one()
+    assert post.post_title == "Set 1"
+    assert "big version" in (post.description or "")
+
+    source = importer.session.execute(
+        select(Source).where(Source.id == post.source_id)
+    ).scalar_one()
+    assert source.platform == "patreon"
+
+    prov_count = importer.session.execute(
+        select(func.count(ImageProvenance.id))
+        .where(ImageProvenance.image_record_id == eid)
+        .where(ImageProvenance.post_id == post.id)
+    ).scalar_one()
+    assert prov_count == 1
+
+
 def test_threshold_controls_match(importer, import_layout):
     # Structurally distinct images (orthogonal splits) are far apart in
     # phash space, so a tight threshold keeps them independent rather than