feat(importer): _supersede() now applies the new (larger) file's sidecar — operator wanted to scan GS download dir to supersede smaller IR-migrated images AND wire up gallery-dl Post metadata, but supersede was file-only and silently dropped the sidecar.
_apply_sidecar is additive: it find-or-creates Post/Source/ImageProvenance and sets primary_post_id NULL-only, so any IR-migration provenance on the existing row survives untouched and the new GS sidecar adds a second ImageProvenance pointing at the freshly-created Post. Wrapped in try/except so a malformed sidecar can't unwind the file-swap commit — the file replacement is the critical operation. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -722,6 +722,15 @@ class Importer:
|
|||||||
row id (so tags/series/curation stay attached). ML is cleared so
|
row id (so tags/series/curation stay attached). ML is cleared so
|
||||||
the import task re-derives it on the new pixels.
|
the import task re-derives it on the new pixels.
|
||||||
|
|
||||||
|
After the file swap, the new file's adjacent gallery-dl sidecar
|
||||||
|
(if any) is applied via _apply_sidecar — operator-flagged
|
||||||
|
2026-05-25: scanning a GS download dir with smaller IR-migrated
|
||||||
|
images on the receiving end used to swap files but lose the GS
|
||||||
|
sidecar's post metadata entirely. _apply_sidecar is additive
|
||||||
|
(find-or-create Post / Source / ImageProvenance, NULL-only
|
||||||
|
primary_post_id update) so any pre-existing Post linkage
|
||||||
|
survives untouched.
|
||||||
|
|
||||||
If `new_path` is provided, `source` is assumed to ALREADY be at
|
If `new_path` is provided, `source` is assumed to ALREADY be at
|
||||||
that path (FC-3c attach_in_place case) — skip the copy step.
|
that path (FC-3c attach_in_place case) — skip the copy step.
|
||||||
Otherwise the file is copied via _copy_to_library."""
|
Otherwise the file is copied via _copy_to_library."""
|
||||||
@@ -751,6 +760,22 @@ class Importer:
|
|||||||
self.session.flush()
|
self.session.flush()
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
|
|
||||||
|
# Sidecar enrichment from the new (larger) file's location.
|
||||||
|
# _apply_sidecar resolves artist from the sidecar itself if the
|
||||||
|
# existing row has none, and is internally guarded against
|
||||||
|
# missing-or-malformed sidecars (silent return).
|
||||||
|
try:
|
||||||
|
self._apply_sidecar(existing, source, None)
|
||||||
|
except Exception as exc:
|
||||||
|
# Don't unwind the supersede DB swap if sidecar parsing
|
||||||
|
# blows up unexpectedly — the file replacement is the
|
||||||
|
# critical operation, sidecar is enrichment.
|
||||||
|
log.warning(
|
||||||
|
"sidecar enrichment failed during supersede of "
|
||||||
|
"image_record.id=%s from %s: %s",
|
||||||
|
existing.id, source, exc,
|
||||||
|
)
|
||||||
|
|
||||||
for stale in (old_path, old_thumb):
|
for stale in (old_path, old_thumb):
|
||||||
if not stale or stale == str(dest):
|
if not stale or stale == str(dest):
|
||||||
# If the supersede kept the file in place (new_path == old
|
# If the supersede kept the file in place (new_path == old
|
||||||
|
|||||||
@@ -10,7 +10,15 @@ import pytest
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
from backend.app.models import ImageRecord, ImportSettings, Tag, TagKind
|
from backend.app.models import (
|
||||||
|
ImageProvenance,
|
||||||
|
ImageRecord,
|
||||||
|
ImportSettings,
|
||||||
|
Post,
|
||||||
|
Source,
|
||||||
|
Tag,
|
||||||
|
TagKind,
|
||||||
|
)
|
||||||
from backend.app.models.tag import image_tag
|
from backend.app.models.tag import image_tag
|
||||||
from backend.app.services.importer import Importer, SkipReason
|
from backend.app.services.importer import Importer, SkipReason
|
||||||
from backend.app.services.thumbnailer import Thumbnailer
|
from backend.app.services.thumbnailer import Thumbnailer
|
||||||
@@ -141,6 +149,67 @@ def test_smaller_existing_is_superseded(importer, import_layout):
|
|||||||
assert Path(row.path).exists()
|
assert Path(row.path).exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_supersede_applies_new_file_sidecar(importer, import_layout):
|
||||||
|
"""Operator-flagged 2026-05-25: scanning the GS download dir should
|
||||||
|
supersede smaller IR-migrated images AND wire up the GS sidecar's
|
||||||
|
Post/Source/ImageProvenance. Previously _supersede swapped the file
|
||||||
|
but ignored the sidecar entirely."""
|
||||||
|
import json
|
||||||
|
import_root, _ = import_layout
|
||||||
|
|
||||||
|
# Stage 1: a small, sidecar-less image (the "IR migration" precondition).
|
||||||
|
small = import_root / "ir-migration-folder" / "small.png"
|
||||||
|
_write(small, (60, 130, 200), (200, 200))
|
||||||
|
r1 = importer.import_one(small)
|
||||||
|
assert r1.status == "imported"
|
||||||
|
eid = r1.image_id
|
||||||
|
|
||||||
|
# Stage 2: a larger version of the same image (same phash) WITH a
|
||||||
|
# gallery-dl JSON sidecar adjacent. Live in a separate folder to
|
||||||
|
# simulate the GS download dir.
|
||||||
|
big = import_root / "Maewix" / "patreon" / "01_big.png"
|
||||||
|
_write(big, (60, 130, 200), (900, 900))
|
||||||
|
sidecar_path = big.with_suffix(big.suffix + ".json")
|
||||||
|
sidecar_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
sidecar_path.write_text(json.dumps({
|
||||||
|
"category": "patreon",
|
||||||
|
"id": 555,
|
||||||
|
"url": "https://www.patreon.com/posts/555",
|
||||||
|
"title": "Set 1",
|
||||||
|
"content": "<p>The big version</p>",
|
||||||
|
"page_count": 1,
|
||||||
|
"published_at": "2025-08-01T00:00:00+00:00",
|
||||||
|
"artist": "Maewix",
|
||||||
|
}))
|
||||||
|
|
||||||
|
r2 = importer.import_one(big)
|
||||||
|
assert r2.status == "superseded"
|
||||||
|
assert r2.image_id == eid
|
||||||
|
|
||||||
|
importer.session.expire_all()
|
||||||
|
# Row preserved, file replaced, sidecar metadata wired up.
|
||||||
|
row = importer.session.get(ImageRecord, eid)
|
||||||
|
assert row.width == 900 and row.height == 900
|
||||||
|
|
||||||
|
post = importer.session.execute(
|
||||||
|
select(Post).where(Post.external_post_id == "555")
|
||||||
|
).scalar_one()
|
||||||
|
assert post.post_title == "Set 1"
|
||||||
|
assert "big version" in (post.description or "")
|
||||||
|
|
||||||
|
source = importer.session.execute(
|
||||||
|
select(Source).where(Source.id == post.source_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert source.platform == "patreon"
|
||||||
|
|
||||||
|
prov_count = importer.session.execute(
|
||||||
|
select(func.count(ImageProvenance.id))
|
||||||
|
.where(ImageProvenance.image_record_id == eid)
|
||||||
|
.where(ImageProvenance.post_id == post.id)
|
||||||
|
).scalar_one()
|
||||||
|
assert prov_count == 1
|
||||||
|
|
||||||
|
|
||||||
def test_threshold_controls_match(importer, import_layout):
|
def test_threshold_controls_match(importer, import_layout):
|
||||||
# Structurally distinct images (orthogonal splits) are far apart in
|
# Structurally distinct images (orthogonal splits) are far apart in
|
||||||
# phash space, so a tight threshold keeps them independent rather than
|
# phash space, so a tight threshold keeps them independent rather than
|
||||||
|
|||||||
Reference in New Issue
Block a user