From 5666fd5ca5e5a30680136000769b5a73b48684c4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 25 May 2026 13:06:11 -0400 Subject: [PATCH 1/4] =?UTF-8?q?fix(ui):=20scan=20trigger=20immediate-feedb?= =?UTF-8?q?ack=20toast=20+=20delayed=20status=20re-poll=20=E2=80=94=20oper?= =?UTF-8?q?ator-flagged=20'click=20does=20nothing'=20was=20actually=20scan?= =?UTF-8?q?=5Fdirectory's=20skip-set=20finalizing=20the=20batch=20in=20<10?= =?UTF-8?q?0ms=20when=20every=20file=20already=20had=20an=20ImportTask=20r?= =?UTF-8?q?ow,=20before=20refreshStatus=20could=20ever=20see=20the=20activ?= =?UTF-8?q?e=20state.=20Now=20the=20click=20always=20produces=20visible=20?= =?UTF-8?q?feedback=20(immediate=20'Scan=20triggered'=20+=202s=20'no=20new?= =?UTF-8?q?=20files'=20if=20it=20quick-finalizes).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/stores/import.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/frontend/src/stores/import.js b/frontend/src/stores/import.js index 9d4832e..2358274 100644 --- a/frontend/src/stores/import.js +++ b/frontend/src/stores/import.js @@ -56,7 +56,27 @@ export const useImportStore = defineStore('import', () => { triggerError.value = null try { await api.post('/api/import/trigger', { body: { mode: 'quick' } }) + // Acknowledge immediately so the click isn't invisible. scan_directory + // can finalize the batch synchronously when every file in /import is + // already on a non-failed ImportTask (operator-flagged 2026-05-25: + // 233k existing tasks → all paths in skip-set → files_seen=0 → + // batch flashes 'running' for <100ms then 'complete' before the + // first refreshStatus() lands; UI never sees the active state). + window.__fcToast?.({ text: 'Scan triggered', type: 'success' }) await refreshStatus() + // Re-poll twice over ~5s to catch quick-finalize transitions and + // surface a result toast either way. + setTimeout(async () => { + await refreshStatus() + if (!activeBatch.value) { + // Either scan completed with zero new files, or it never visibly + // started. Fetch the freshest task to differentiate. + window.__fcToast?.({ + text: 'Scan complete — no new files (everything already on an ImportTask row)', + type: 'info', + }) + } + }, 2000) } catch (e) { triggerError.value = e.message window.__fcToast?.({ text: `Scan failed: ${e.message}`, type: 'error' }) From 1bbe478fd0a3d9f86ad479b42a3b244164b25441 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 25 May 2026 13:09:25 -0400 Subject: [PATCH 2/4] =?UTF-8?q?ci:=20report=20slowest=2025=20integration?= =?UTF-8?q?=20tests=20via=20pytest=20--durations=3D25=20=E2=80=94=20instru?= =?UTF-8?q?mentation=20pass=20before=20deciding=20parallelization=20vs=20t?= =?UTF-8?q?argeted=20slow-test=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .forgejo/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 829aa4c..4687738 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -121,4 +121,4 @@ jobs: done pip install -r requirements.txt pytest pytest-asyncio alembic upgrade head - pytest tests/ -v -m integration + pytest tests/ -v -m integration --durations=25 From b0bb7ae6cc5f84cb0cd15aec84dbf02b7dc488fe Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 25 May 2026 13:47:27 -0400 Subject: [PATCH 3/4] =?UTF-8?q?ci:=20pip=20wheel=20cache=20(actions/cache?= =?UTF-8?q?=20on=20requirements.txt=20hash)=20+=20uv-when-available=20?= =?UTF-8?q?=E2=80=94=20~2=20min=20saved=20on=20warm=20runs,=20no=20risk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uv falls back to pip install on runners without uv binary, so this change is forward-compatible with the current ci-python image. When the runner image gets uv pre-installed in a future bump, the warm install path drops from ~2 min to ~10 seconds. pytest-xdist parallelization is OUT OF SCOPE for this commit: tests/conftest.py uses a TRUNCATE ALL TABLES RESTART IDENTITY CASCADE fixture after every integration test against a single shared database; xdist workers running in parallel would nuke each other's mid-test state. A future refactor to per-worker databases or per-worker schema isolation is the prerequisite. Co-Authored-By: Claude Opus 4.7 (1M context) --- .forgejo/workflows/ci.yml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 4687738..6402f01 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -24,11 +24,26 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Cache pip wheels + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} + restore-keys: | + pip-${{ runner.os }}-py314- + - name: Install Python deps # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/ # Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain # versions live on the runner image, not here. - run: pip install -r requirements.txt pytest pytest-asyncio + # uv: 5-10x faster wheel resolve than pip for cold caches. + # Falls back to pip install on uv-missing runners (older images). + run: | + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi - name: Ruff lint run: ruff check backend/ tests/ alembic/ @@ -99,6 +114,14 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Cache pip wheels + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} + restore-keys: | + pip-${{ runner.os }}-py314- + - name: Integration suite (resolve service IPs, migrate, test) run: | set -eux @@ -119,6 +142,11 @@ jobs: (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break sleep 2 done - pip install -r requirements.txt pytest pytest-asyncio + # uv when available (5-10x faster wheel resolve); fall back to pip. + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi alembic upgrade head pytest tests/ -v -m integration --durations=25 From a0470b5f607c9f18b616a95777ae27707c7ff479 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 25 May 2026 13:51:51 -0400 Subject: [PATCH 4/4] =?UTF-8?q?feat(importer):=20=5Fsupersede()=20now=20ap?= =?UTF-8?q?plies=20the=20new=20(larger)=20file's=20sidecar=20=E2=80=94=20o?= =?UTF-8?q?perator=20wanted=20to=20scan=20GS=20download=20dir=20to=20super?= =?UTF-8?q?sede=20smaller=20IR-migrated=20images=20AND=20wire=20up=20galle?= =?UTF-8?q?ry-dl=20Post=20metadata,=20but=20supersede=20was=20file-only=20?= =?UTF-8?q?and=20silently=20dropped=20the=20sidecar.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _apply_sidecar is additive: it find-or-creates Post/Source/ImageProvenance and sets primary_post_id NULL-only, so any IR-migration provenance on the existing row survives untouched and the new GS sidecar adds a second ImageProvenance pointing at the freshly-created Post. Wrapped in try/except so a malformed sidecar can't unwind the file-swap commit — the file replacement is the critical operation. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/importer.py | 25 +++++++++++ tests/test_phash_dedup.py | 71 +++++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 2548d5b..f9eb62b 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -722,6 +722,15 @@ class Importer: row id (so tags/series/curation stay attached). ML is cleared so the import task re-derives it on the new pixels. + After the file swap, the new file's adjacent gallery-dl sidecar + (if any) is applied via _apply_sidecar — operator-flagged + 2026-05-25: scanning a GS download dir with smaller IR-migrated + images on the receiving end used to swap files but lose the GS + sidecar's post metadata entirely. _apply_sidecar is additive + (find-or-create Post / Source / ImageProvenance, NULL-only + primary_post_id update) so any pre-existing Post linkage + survives untouched. + If `new_path` is provided, `source` is assumed to ALREADY be at that path (FC-3c attach_in_place case) — skip the copy step. Otherwise the file is copied via _copy_to_library.""" @@ -751,6 +760,22 @@ class Importer: self.session.flush() self.session.commit() + # Sidecar enrichment from the new (larger) file's location. + # _apply_sidecar resolves artist from the sidecar itself if the + # existing row has none, and is internally guarded against + # missing-or-malformed sidecars (silent return). + try: + self._apply_sidecar(existing, source, None) + except Exception as exc: + # Don't unwind the supersede DB swap if sidecar parsing + # blows up unexpectedly — the file replacement is the + # critical operation, sidecar is enrichment. + log.warning( + "sidecar enrichment failed during supersede of " + "image_record.id=%s from %s: %s", + existing.id, source, exc, + ) + for stale in (old_path, old_thumb): if not stale or stale == str(dest): # If the supersede kept the file in place (new_path == old diff --git a/tests/test_phash_dedup.py b/tests/test_phash_dedup.py index 3aeebce..55fde03 100644 --- a/tests/test_phash_dedup.py +++ b/tests/test_phash_dedup.py @@ -10,7 +10,15 @@ import pytest from PIL import Image from sqlalchemy import func, select -from backend.app.models import ImageRecord, ImportSettings, Tag, TagKind +from backend.app.models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + Source, + Tag, + TagKind, +) from backend.app.models.tag import image_tag from backend.app.services.importer import Importer, SkipReason from backend.app.services.thumbnailer import Thumbnailer @@ -141,6 +149,67 @@ def test_smaller_existing_is_superseded(importer, import_layout): assert Path(row.path).exists() +def test_supersede_applies_new_file_sidecar(importer, import_layout): + """Operator-flagged 2026-05-25: scanning the GS download dir should + supersede smaller IR-migrated images AND wire up the GS sidecar's + Post/Source/ImageProvenance. Previously _supersede swapped the file + but ignored the sidecar entirely.""" + import json + import_root, _ = import_layout + + # Stage 1: a small, sidecar-less image (the "IR migration" precondition). + small = import_root / "ir-migration-folder" / "small.png" + _write(small, (60, 130, 200), (200, 200)) + r1 = importer.import_one(small) + assert r1.status == "imported" + eid = r1.image_id + + # Stage 2: a larger version of the same image (same phash) WITH a + # gallery-dl JSON sidecar adjacent. Live in a separate folder to + # simulate the GS download dir. + big = import_root / "Maewix" / "patreon" / "01_big.png" + _write(big, (60, 130, 200), (900, 900)) + sidecar_path = big.with_suffix(big.suffix + ".json") + sidecar_path.parent.mkdir(parents=True, exist_ok=True) + sidecar_path.write_text(json.dumps({ + "category": "patreon", + "id": 555, + "url": "https://www.patreon.com/posts/555", + "title": "Set 1", + "content": "

The big version

", + "page_count": 1, + "published_at": "2025-08-01T00:00:00+00:00", + "artist": "Maewix", + })) + + r2 = importer.import_one(big) + assert r2.status == "superseded" + assert r2.image_id == eid + + importer.session.expire_all() + # Row preserved, file replaced, sidecar metadata wired up. + row = importer.session.get(ImageRecord, eid) + assert row.width == 900 and row.height == 900 + + post = importer.session.execute( + select(Post).where(Post.external_post_id == "555") + ).scalar_one() + assert post.post_title == "Set 1" + assert "big version" in (post.description or "") + + source = importer.session.execute( + select(Source).where(Source.id == post.source_id) + ).scalar_one() + assert source.platform == "patreon" + + prov_count = importer.session.execute( + select(func.count(ImageProvenance.id)) + .where(ImageProvenance.image_record_id == eid) + .where(ImageProvenance.post_id == post.id) + ).scalar_one() + assert prov_count == 1 + + def test_threshold_controls_match(importer, import_layout): # Structurally distinct images (orthogonal splits) are far apart in # phash space, so a tight threshold keeps them independent rather than