feat(importer): _supersede() now applies the new (larger) file's sidecar — operator wanted to scan GS download dir to supersede smaller IR-migrated images AND wire up gallery-dl Post metadata, but supersede was file-only and silently dropped the sidecar.
_apply_sidecar is additive: it find-or-creates Post/Source/ImageProvenance and sets primary_post_id NULL-only, so any IR-migration provenance on the existing row survives untouched and the new GS sidecar adds a second ImageProvenance pointing at the freshly-created Post. Wrapped in try/except so a malformed sidecar can't unwind the file-swap commit — the file replacement is the critical operation. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -722,6 +722,15 @@ class Importer:
|
||||
row id (so tags/series/curation stay attached). ML is cleared so
|
||||
the import task re-derives it on the new pixels.
|
||||
|
||||
After the file swap, the new file's adjacent gallery-dl sidecar
|
||||
(if any) is applied via _apply_sidecar — operator-flagged
|
||||
2026-05-25: scanning a GS download dir with smaller IR-migrated
|
||||
images on the receiving end used to swap files but lose the GS
|
||||
sidecar's post metadata entirely. _apply_sidecar is additive
|
||||
(find-or-create Post / Source / ImageProvenance, NULL-only
|
||||
primary_post_id update) so any pre-existing Post linkage
|
||||
survives untouched.
|
||||
|
||||
If `new_path` is provided, `source` is assumed to ALREADY be at
|
||||
that path (FC-3c attach_in_place case) — skip the copy step.
|
||||
Otherwise the file is copied via _copy_to_library."""
|
||||
@@ -751,6 +760,22 @@ class Importer:
|
||||
self.session.flush()
|
||||
self.session.commit()
|
||||
|
||||
# Sidecar enrichment from the new (larger) file's location.
|
||||
# _apply_sidecar resolves artist from the sidecar itself if the
|
||||
# existing row has none, and is internally guarded against
|
||||
# missing-or-malformed sidecars (silent return).
|
||||
try:
|
||||
self._apply_sidecar(existing, source, None)
|
||||
except Exception as exc:
|
||||
# Don't unwind the supersede DB swap if sidecar parsing
|
||||
# blows up unexpectedly — the file replacement is the
|
||||
# critical operation, sidecar is enrichment.
|
||||
log.warning(
|
||||
"sidecar enrichment failed during supersede of "
|
||||
"image_record.id=%s from %s: %s",
|
||||
existing.id, source, exc,
|
||||
)
|
||||
|
||||
for stale in (old_path, old_thumb):
|
||||
if not stale or stale == str(dest):
|
||||
# If the supersede kept the file in place (new_path == old
|
||||
|
||||
@@ -10,7 +10,15 @@ import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import ImageRecord, ImportSettings, Tag, TagKind
|
||||
from backend.app.models import (
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
Post,
|
||||
Source,
|
||||
Tag,
|
||||
TagKind,
|
||||
)
|
||||
from backend.app.models.tag import image_tag
|
||||
from backend.app.services.importer import Importer, SkipReason
|
||||
from backend.app.services.thumbnailer import Thumbnailer
|
||||
@@ -141,6 +149,67 @@ def test_smaller_existing_is_superseded(importer, import_layout):
|
||||
assert Path(row.path).exists()
|
||||
|
||||
|
||||
def test_supersede_applies_new_file_sidecar(importer, import_layout):
|
||||
"""Operator-flagged 2026-05-25: scanning the GS download dir should
|
||||
supersede smaller IR-migrated images AND wire up the GS sidecar's
|
||||
Post/Source/ImageProvenance. Previously _supersede swapped the file
|
||||
but ignored the sidecar entirely."""
|
||||
import json
|
||||
import_root, _ = import_layout
|
||||
|
||||
# Stage 1: a small, sidecar-less image (the "IR migration" precondition).
|
||||
small = import_root / "ir-migration-folder" / "small.png"
|
||||
_write(small, (60, 130, 200), (200, 200))
|
||||
r1 = importer.import_one(small)
|
||||
assert r1.status == "imported"
|
||||
eid = r1.image_id
|
||||
|
||||
# Stage 2: a larger version of the same image (same phash) WITH a
|
||||
# gallery-dl JSON sidecar adjacent. Live in a separate folder to
|
||||
# simulate the GS download dir.
|
||||
big = import_root / "Maewix" / "patreon" / "01_big.png"
|
||||
_write(big, (60, 130, 200), (900, 900))
|
||||
sidecar_path = big.with_suffix(big.suffix + ".json")
|
||||
sidecar_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
sidecar_path.write_text(json.dumps({
|
||||
"category": "patreon",
|
||||
"id": 555,
|
||||
"url": "https://www.patreon.com/posts/555",
|
||||
"title": "Set 1",
|
||||
"content": "<p>The big version</p>",
|
||||
"page_count": 1,
|
||||
"published_at": "2025-08-01T00:00:00+00:00",
|
||||
"artist": "Maewix",
|
||||
}))
|
||||
|
||||
r2 = importer.import_one(big)
|
||||
assert r2.status == "superseded"
|
||||
assert r2.image_id == eid
|
||||
|
||||
importer.session.expire_all()
|
||||
# Row preserved, file replaced, sidecar metadata wired up.
|
||||
row = importer.session.get(ImageRecord, eid)
|
||||
assert row.width == 900 and row.height == 900
|
||||
|
||||
post = importer.session.execute(
|
||||
select(Post).where(Post.external_post_id == "555")
|
||||
).scalar_one()
|
||||
assert post.post_title == "Set 1"
|
||||
assert "big version" in (post.description or "")
|
||||
|
||||
source = importer.session.execute(
|
||||
select(Source).where(Source.id == post.source_id)
|
||||
).scalar_one()
|
||||
assert source.platform == "patreon"
|
||||
|
||||
prov_count = importer.session.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
.where(ImageProvenance.image_record_id == eid)
|
||||
.where(ImageProvenance.post_id == post.id)
|
||||
).scalar_one()
|
||||
assert prov_count == 1
|
||||
|
||||
|
||||
def test_threshold_controls_match(importer, import_layout):
|
||||
# Structurally distinct images (orthogonal splits) are far apart in
|
||||
# phash space, so a tight threshold keeps them independent rather than
|
||||
|
||||
Reference in New Issue
Block a user