feat(importer): _supersede() now applies the new (larger) file's sidecar — operator wanted to scan GS download dir to supersede smaller IR-migrated images AND wire up gallery-dl Post metadata, but supersede was file-only and silently dropped the sidecar.

_apply_sidecar is additive: it find-or-creates Post/Source/ImageProvenance
and sets primary_post_id NULL-only, so any IR-migration provenance on the
existing row survives untouched and the new GS sidecar adds a second
ImageProvenance pointing at the freshly-created Post.

Wrapped in try/except so a malformed sidecar can't unwind the file-swap
commit — the file replacement is the critical operation.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-25 13:51:51 -04:00
parent b0bb7ae6cc
commit a0470b5f60
2 changed files with 95 additions and 1 deletions
+25
View File
@@ -722,6 +722,15 @@ class Importer:
row id (so tags/series/curation stay attached). ML is cleared so
the import task re-derives it on the new pixels.
After the file swap, the new file's adjacent gallery-dl sidecar
(if any) is applied via _apply_sidecar — operator-flagged
2026-05-25: scanning a GS download dir with smaller IR-migrated
images on the receiving end used to swap files but lose the GS
sidecar's post metadata entirely. _apply_sidecar is additive
(find-or-create Post / Source / ImageProvenance, NULL-only
primary_post_id update) so any pre-existing Post linkage
survives untouched.
If `new_path` is provided, `source` is assumed to ALREADY be at
that path (FC-3c attach_in_place case) — skip the copy step.
Otherwise the file is copied via _copy_to_library."""
@@ -751,6 +760,22 @@ class Importer:
self.session.flush()
self.session.commit()
# Sidecar enrichment from the new (larger) file's location.
# _apply_sidecar resolves artist from the sidecar itself if the
# existing row has none, and is internally guarded against
# missing-or-malformed sidecars (silent return).
try:
self._apply_sidecar(existing, source, None)
except Exception as exc:
# Don't unwind the supersede DB swap if sidecar parsing
# blows up unexpectedly — the file replacement is the
# critical operation, sidecar is enrichment.
log.warning(
"sidecar enrichment failed during supersede of "
"image_record.id=%s from %s: %s",
existing.id, source, exc,
)
for stale in (old_path, old_thumb):
if not stale or stale == str(dest):
# If the supersede kept the file in place (new_path == old
+70 -1
View File
@@ -10,7 +10,15 @@ import pytest
from PIL import Image
from sqlalchemy import func, select
from backend.app.models import ImageRecord, ImportSettings, Tag, TagKind
from backend.app.models import (
ImageProvenance,
ImageRecord,
ImportSettings,
Post,
Source,
Tag,
TagKind,
)
from backend.app.models.tag import image_tag
from backend.app.services.importer import Importer, SkipReason
from backend.app.services.thumbnailer import Thumbnailer
@@ -141,6 +149,67 @@ def test_smaller_existing_is_superseded(importer, import_layout):
assert Path(row.path).exists()
def test_supersede_applies_new_file_sidecar(importer, import_layout):
"""Operator-flagged 2026-05-25: scanning the GS download dir should
supersede smaller IR-migrated images AND wire up the GS sidecar's
Post/Source/ImageProvenance. Previously _supersede swapped the file
but ignored the sidecar entirely."""
import json
import_root, _ = import_layout
# Stage 1: a small, sidecar-less image (the "IR migration" precondition).
small = import_root / "ir-migration-folder" / "small.png"
_write(small, (60, 130, 200), (200, 200))
r1 = importer.import_one(small)
assert r1.status == "imported"
eid = r1.image_id
# Stage 2: a larger version of the same image (same phash) WITH a
# gallery-dl JSON sidecar adjacent. Live in a separate folder to
# simulate the GS download dir.
big = import_root / "Maewix" / "patreon" / "01_big.png"
_write(big, (60, 130, 200), (900, 900))
sidecar_path = big.with_suffix(big.suffix + ".json")
sidecar_path.parent.mkdir(parents=True, exist_ok=True)
sidecar_path.write_text(json.dumps({
"category": "patreon",
"id": 555,
"url": "https://www.patreon.com/posts/555",
"title": "Set 1",
"content": "<p>The big version</p>",
"page_count": 1,
"published_at": "2025-08-01T00:00:00+00:00",
"artist": "Maewix",
}))
r2 = importer.import_one(big)
assert r2.status == "superseded"
assert r2.image_id == eid
importer.session.expire_all()
# Row preserved, file replaced, sidecar metadata wired up.
row = importer.session.get(ImageRecord, eid)
assert row.width == 900 and row.height == 900
post = importer.session.execute(
select(Post).where(Post.external_post_id == "555")
).scalar_one()
assert post.post_title == "Set 1"
assert "big version" in (post.description or "")
source = importer.session.execute(
select(Source).where(Source.id == post.source_id)
).scalar_one()
assert source.platform == "patreon"
prov_count = importer.session.execute(
select(func.count(ImageProvenance.id))
.where(ImageProvenance.image_record_id == eid)
.where(ImageProvenance.post_id == post.id)
).scalar_one()
assert prov_count == 1
def test_threshold_controls_match(importer, import_layout):
# Structurally distinct images (orthogonal splits) are far apart in
# phash space, so a tight threshold keeps them independent rather than