diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 829aa4c..6402f01 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -24,11 +24,26 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Cache pip wheels + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} + restore-keys: | + pip-${{ runner.os }}-py314- + - name: Install Python deps # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/ # Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain # versions live on the runner image, not here. - run: pip install -r requirements.txt pytest pytest-asyncio + # uv: 5-10x faster wheel resolve than pip for cold caches. + # Falls back to pip install on uv-missing runners (older images). + run: | + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi - name: Ruff lint run: ruff check backend/ tests/ alembic/ @@ -99,6 +114,14 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Cache pip wheels + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} + restore-keys: | + pip-${{ runner.os }}-py314- + - name: Integration suite (resolve service IPs, migrate, test) run: | set -eux @@ -119,6 +142,11 @@ jobs: (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break sleep 2 done - pip install -r requirements.txt pytest pytest-asyncio + # uv when available (5-10x faster wheel resolve); fall back to pip. + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi alembic upgrade head - pytest tests/ -v -m integration + pytest tests/ -v -m integration --durations=25 diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 2548d5b..f9eb62b 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -722,6 +722,15 @@ class Importer: row id (so tags/series/curation stay attached). ML is cleared so the import task re-derives it on the new pixels. + After the file swap, the new file's adjacent gallery-dl sidecar + (if any) is applied via _apply_sidecar — operator-flagged + 2026-05-25: scanning a GS download dir with smaller IR-migrated + images on the receiving end used to swap files but lose the GS + sidecar's post metadata entirely. _apply_sidecar is additive + (find-or-create Post / Source / ImageProvenance, NULL-only + primary_post_id update) so any pre-existing Post linkage + survives untouched. + If `new_path` is provided, `source` is assumed to ALREADY be at that path (FC-3c attach_in_place case) — skip the copy step. Otherwise the file is copied via _copy_to_library.""" @@ -751,6 +760,22 @@ class Importer: self.session.flush() self.session.commit() + # Sidecar enrichment from the new (larger) file's location. + # _apply_sidecar resolves artist from the sidecar itself if the + # existing row has none, and is internally guarded against + # missing-or-malformed sidecars (silent return). + try: + self._apply_sidecar(existing, source, None) + except Exception as exc: + # Don't unwind the supersede DB swap if sidecar parsing + # blows up unexpectedly — the file replacement is the + # critical operation, sidecar is enrichment. + log.warning( + "sidecar enrichment failed during supersede of " + "image_record.id=%s from %s: %s", + existing.id, source, exc, + ) + for stale in (old_path, old_thumb): if not stale or stale == str(dest): # If the supersede kept the file in place (new_path == old diff --git a/frontend/src/stores/import.js b/frontend/src/stores/import.js index 9d4832e..2358274 100644 --- a/frontend/src/stores/import.js +++ b/frontend/src/stores/import.js @@ -56,7 +56,27 @@ export const useImportStore = defineStore('import', () => { triggerError.value = null try { await api.post('/api/import/trigger', { body: { mode: 'quick' } }) + // Acknowledge immediately so the click isn't invisible. scan_directory + // can finalize the batch synchronously when every file in /import is + // already on a non-failed ImportTask (operator-flagged 2026-05-25: + // 233k existing tasks → all paths in skip-set → files_seen=0 → + // batch flashes 'running' for <100ms then 'complete' before the + // first refreshStatus() lands; UI never sees the active state). + window.__fcToast?.({ text: 'Scan triggered', type: 'success' }) await refreshStatus() + // Re-poll twice over ~5s to catch quick-finalize transitions and + // surface a result toast either way. + setTimeout(async () => { + await refreshStatus() + if (!activeBatch.value) { + // Either scan completed with zero new files, or it never visibly + // started. Fetch the freshest task to differentiate. + window.__fcToast?.({ + text: 'Scan complete — no new files (everything already on an ImportTask row)', + type: 'info', + }) + } + }, 2000) } catch (e) { triggerError.value = e.message window.__fcToast?.({ text: `Scan failed: ${e.message}`, type: 'error' }) diff --git a/tests/test_phash_dedup.py b/tests/test_phash_dedup.py index 3aeebce..55fde03 100644 --- a/tests/test_phash_dedup.py +++ b/tests/test_phash_dedup.py @@ -10,7 +10,15 @@ import pytest from PIL import Image from sqlalchemy import func, select -from backend.app.models import ImageRecord, ImportSettings, Tag, TagKind +from backend.app.models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + Source, + Tag, + TagKind, +) from backend.app.models.tag import image_tag from backend.app.services.importer import Importer, SkipReason from backend.app.services.thumbnailer import Thumbnailer @@ -141,6 +149,67 @@ def test_smaller_existing_is_superseded(importer, import_layout): assert Path(row.path).exists() +def test_supersede_applies_new_file_sidecar(importer, import_layout): + """Operator-flagged 2026-05-25: scanning the GS download dir should + supersede smaller IR-migrated images AND wire up the GS sidecar's + Post/Source/ImageProvenance. Previously _supersede swapped the file + but ignored the sidecar entirely.""" + import json + import_root, _ = import_layout + + # Stage 1: a small, sidecar-less image (the "IR migration" precondition). + small = import_root / "ir-migration-folder" / "small.png" + _write(small, (60, 130, 200), (200, 200)) + r1 = importer.import_one(small) + assert r1.status == "imported" + eid = r1.image_id + + # Stage 2: a larger version of the same image (same phash) WITH a + # gallery-dl JSON sidecar adjacent. Live in a separate folder to + # simulate the GS download dir. + big = import_root / "Maewix" / "patreon" / "01_big.png" + _write(big, (60, 130, 200), (900, 900)) + sidecar_path = big.with_suffix(big.suffix + ".json") + sidecar_path.parent.mkdir(parents=True, exist_ok=True) + sidecar_path.write_text(json.dumps({ + "category": "patreon", + "id": 555, + "url": "https://www.patreon.com/posts/555", + "title": "Set 1", + "content": "

The big version

", + "page_count": 1, + "published_at": "2025-08-01T00:00:00+00:00", + "artist": "Maewix", + })) + + r2 = importer.import_one(big) + assert r2.status == "superseded" + assert r2.image_id == eid + + importer.session.expire_all() + # Row preserved, file replaced, sidecar metadata wired up. + row = importer.session.get(ImageRecord, eid) + assert row.width == 900 and row.height == 900 + + post = importer.session.execute( + select(Post).where(Post.external_post_id == "555") + ).scalar_one() + assert post.post_title == "Set 1" + assert "big version" in (post.description or "") + + source = importer.session.execute( + select(Source).where(Source.id == post.source_id) + ).scalar_one() + assert source.platform == "patreon" + + prov_count = importer.session.execute( + select(func.count(ImageProvenance.id)) + .where(ImageProvenance.image_record_id == eid) + .where(ImageProvenance.post_id == post.id) + ).scalar_one() + assert prov_count == 1 + + def test_threshold_controls_match(importer, import_layout): # Structurally distinct images (orthogonal splits) are far apart in # phash space, so a tight threshold keeps them independent rather than