From efcb548ebf06a2926564e966130d32285eda9fa9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 25 Sep 2026 09:30:17 -0400 Subject: [PATCH] fix: natively downloaded images take their post's date, not their download time (#4431) The native ingesters (Discord, Patreon, SubscribeStar) import a post's media before its record. Only the record (`_post.json`) carries the date: the Discord message timestamp, or the Patreon/SubscribeStar published_at. So each image was linked to a post with no date yet, and it kept its download time in both gallery date columns. The post itself was dated correctly once the record landed, but nothing went back to update its images. - upsert_post_record now re-dates the images already linked to the post. `effective_date` becomes the primary post's date, and `earliest_post_date` the earliest dated post the image is in, which are the same rules _attach_provenance applies. - Migration 0113 repairs the library from the posts. It writes only rows that differ. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR --- alembic/versions/0113_redate_native_images.py | 60 ++++++++++++++++ backend/app/services/importer.py | 42 +++++++++++ tests/test_redate_images_migration.py | 70 +++++++++++++++++++ tests/test_sidecar_import.py | 33 +++++++++ 4 files changed, 205 insertions(+) create mode 100644 alembic/versions/0113_redate_native_images.py create mode 100644 tests/test_redate_images_migration.py diff --git a/alembic/versions/0113_redate_native_images.py b/alembic/versions/0113_redate_native_images.py new file mode 100644 index 0000000..1989d41 --- /dev/null +++ b/alembic/versions/0113_redate_native_images.py @@ -0,0 +1,60 @@ +"""Re-date images whose post's date arrived after they were linked. + +#4431. The native ingesters import a post's media before its record, and the +date travels in the record (`_post.json`). Every natively downloaded image was +therefore linked to an undated post and kept its download time in both gallery +date columns, while the post itself was dated correctly. The importer now +re-dates a post's images when its record lands; this repairs the images that +landed before that. + +Both columns get back the rules the importer keeps: + +* `effective_date` is the primary post's date (left alone when that post has + none, as the importer does); +* `earliest_post_date` is the earliest dated post the image is linked to. + +Only rows that differ are written. The downgrade does nothing: the old values +were download times that no one chose. + +Revision ID: 0113 +Revises: 0112 +Create Date: 2026-09-25 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0113" +down_revision = "0112" +branch_labels = None +depends_on = None + + +def redate_images(conn) -> None: + """The data step, on a plain connection, so a test can run it directly.""" + conn.execute(sa.text(""" + UPDATE image_record ir SET effective_date = p.post_date + FROM post p + WHERE p.id = ir.primary_post_id + AND p.post_date IS NOT NULL + AND ir.effective_date IS DISTINCT FROM p.post_date + """)) + conn.execute(sa.text(""" + UPDATE image_record ir SET earliest_post_date = m.earliest + FROM ( + SELECT ip.image_record_id, MIN(p.post_date) AS earliest + FROM image_provenance ip JOIN post p ON p.id = ip.post_id + WHERE p.post_date IS NOT NULL + GROUP BY ip.image_record_id + ) m + WHERE m.image_record_id = ir.id + AND ir.earliest_post_date IS DISTINCT FROM m.earliest + """)) + + +def upgrade(): + redate_images(op.get_bind()) + + +def downgrade(): + pass diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index a7edb5f..8a3b4f4 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -1130,9 +1130,51 @@ class Importer: if post.artist_id is None: post.artist_id = artist.id self._apply_post_fields(post, sd) + self._redate_post_images(post) self.session.commit() return True + def _redate_post_images(self, post: Post) -> None: + """Carry a post's date onto the images already linked to it (#4431). + + The native ingesters import a post's media BEFORE its record: the + per-media sidecar holds only the image identity (post-first, #856), and + the date arrives with `_post.json`. So `_attach_provenance` links each + image to a post that has no date yet, and the image keeps its download + time. This runs when the record lands, and applies the same two rules + `_attach_provenance` applies: `effective_date` is the PRIMARY post's + date, and `earliest_post_date` is the earliest date across every post + the image is linked to. Only rows that differ are written.""" + if post.post_date is None: + return + self.session.flush() + self.session.execute( + update(ImageRecord) + .where(ImageRecord.primary_post_id == post.id) + .where(ImageRecord.effective_date.is_distinct_from(post.post_date)) + .values(effective_date=post.post_date) + .execution_options(synchronize_session=False) + ) + linked = select(ImageProvenance.image_record_id).where( + ImageProvenance.post_id == post.id + ) + earliest = ( + select(func.min(Post.post_date)) + .select_from(ImageProvenance) + .join(Post, Post.id == ImageProvenance.post_id) + .where(ImageProvenance.image_record_id == ImageRecord.id) + .where(Post.post_date.is_not(None)) + .correlate(ImageRecord) + .scalar_subquery() + ) + self.session.execute( + update(ImageRecord) + .where(ImageRecord.id.in_(linked)) + .where(ImageRecord.earliest_post_date.is_distinct_from(earliest)) + .values(earliest_post_date=earliest) + .execution_options(synchronize_session=False) + ) + def attach_in_place( self, path: Path, diff --git a/tests/test_redate_images_migration.py b/tests/test_redate_images_migration.py new file mode 100644 index 0000000..8441923 --- /dev/null +++ b/tests/test_redate_images_migration.py @@ -0,0 +1,70 @@ +"""Migration 0113 (#4431): images linked to a post before its date arrived get +the post's date back. Runs the migration's data step against real rows.""" +import importlib.util +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from backend.app.models import Artist, ImageProvenance, ImageRecord, Post +from tests.factories import make_image as _img + +pytestmark = pytest.mark.integration + +_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "alembic" / "versions" / "0113_redate_native_images.py" +) + + +def _redate(): + spec = importlib.util.spec_from_file_location("m0113", _MIGRATION) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.redate_images + + +def _post(db, artist, epid, when): + p = Post(artist_id=artist.id, external_post_id=epid, post_date=when) + db.add(p) + db.flush() + return p + + +def _link(db, img, post, *, primary): + db.add(ImageProvenance(image_record_id=img.id, post_id=post.id)) + if primary: + img.primary_post_id = post.id + db.flush() + + +def test_redate_images_from_their_posts(db_sync): + sent = datetime(2024, 3, 1, 18, 30, tzinfo=UTC) + earlier = datetime(2023, 1, 5, tzinfo=UTC) + artist = Artist(name="Alice", slug="alice") + db_sync.add(artist) + db_sync.flush() + undated = _post(db_sync, artist, "u1", None) + dated = _post(db_sync, artist, "d1", sent) + repost = _post(db_sync, artist, "r1", earlier) + + stale = _img(db_sync, "a" * 64) # primary post dated, image not + reposted = _img(db_sync, "b" * 64) # also in an earlier post + orphan = _img(db_sync, "c" * 64) # only an undated post + _link(db_sync, stale, dated, primary=True) + _link(db_sync, reposted, dated, primary=True) + _link(db_sync, reposted, repost, primary=False) + _link(db_sync, orphan, undated, primary=True) + orphan_before = orphan.effective_date + + _redate()(db_sync.connection()) + db_sync.expire_all() + + stale = db_sync.get(ImageRecord, stale.id) + reposted = db_sync.get(ImageRecord, reposted.id) + orphan = db_sync.get(ImageRecord, orphan.id) + assert stale.effective_date == sent + assert stale.earliest_post_date == sent + assert reposted.effective_date == sent # the primary post's date + assert reposted.earliest_post_date == earlier # the earliest post's date + assert orphan.effective_date == orphan_before # no date to take diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py index 87f1408..6149624 100644 --- a/tests/test_sidecar_import.py +++ b/tests/test_sidecar_import.py @@ -382,3 +382,36 @@ def test_external_links_not_duplicated_on_reimport(importer, import_layout): assert importer.session.execute( select(func.count()).select_from(ExternalLink) ).scalar_one() == 1 + + +def test_post_record_redates_images_linked_before_it(importer, import_layout): + """#4431: the native ingesters import a message's media before its record, + and only the record carries the date. The images start on their download + time; when the record lands they take the post's date.""" + import_root, _ = import_layout + artist = Artist(name="Alice", slug="alice") + importer.session.add(artist) + importer.session.flush() + m = import_root / "Alice" / "20240301_123_01_art.jpg" + _split(m, "v") + _sidecar(m, {"category": "discord", "message_id": "123"}) + r = importer.import_one(m) + assert r.status == "imported" + rec = importer.session.get(ImageRecord, r.image_id) + post = importer.session.execute(select(Post)).scalar_one() + assert post.post_date is None + download_time = rec.effective_date + + sc = import_root / "Alice" / "20240301_123_post.json" + sc.write_text(json.dumps({ + "category": "discord", "message_id": "123", "message": "", + "date": "2024-03-01T18:30:00.000000+00:00", + })) + assert importer.upsert_post_record(sc, artist=artist) is True + importer.session.expire_all() + rec = importer.session.get(ImageRecord, r.image_id) + post = importer.session.execute(select(Post)).scalar_one() + assert post.post_date is not None + assert post.post_date != download_time + assert rec.effective_date == post.post_date + assert rec.earliest_post_date == post.post_date