diff --git a/alembic/versions/0113_redate_native_images.py b/alembic/versions/0113_redate_native_images.py new file mode 100644 index 0000000..1989d41 --- /dev/null +++ b/alembic/versions/0113_redate_native_images.py @@ -0,0 +1,60 @@ +"""Re-date images whose post's date arrived after they were linked. + +#4431. The native ingesters import a post's media before its record, and the +date travels in the record (`_post.json`). Every natively downloaded image was +therefore linked to an undated post and kept its download time in both gallery +date columns, while the post itself was dated correctly. The importer now +re-dates a post's images when its record lands; this repairs the images that +landed before that. + +Both columns get back the rules the importer keeps: + +* `effective_date` is the primary post's date (left alone when that post has + none, as the importer does); +* `earliest_post_date` is the earliest dated post the image is linked to. + +Only rows that differ are written. The downgrade does nothing: the old values +were download times that no one chose. + +Revision ID: 0113 +Revises: 0112 +Create Date: 2026-09-25 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0113" +down_revision = "0112" +branch_labels = None +depends_on = None + + +def redate_images(conn) -> None: + """The data step, on a plain connection, so a test can run it directly.""" + conn.execute(sa.text(""" + UPDATE image_record ir SET effective_date = p.post_date + FROM post p + WHERE p.id = ir.primary_post_id + AND p.post_date IS NOT NULL + AND ir.effective_date IS DISTINCT FROM p.post_date + """)) + conn.execute(sa.text(""" + UPDATE image_record ir SET earliest_post_date = m.earliest + FROM ( + SELECT ip.image_record_id, MIN(p.post_date) AS earliest + FROM image_provenance ip JOIN post p ON p.id = ip.post_id + WHERE p.post_date IS NOT NULL + GROUP BY ip.image_record_id + ) m + WHERE m.image_record_id = ir.id + AND ir.earliest_post_date IS DISTINCT FROM m.earliest + """)) + + +def upgrade(): + redate_images(op.get_bind()) + + +def downgrade(): + pass diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index a7edb5f..8a3b4f4 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -1130,9 +1130,51 @@ class Importer: if post.artist_id is None: post.artist_id = artist.id self._apply_post_fields(post, sd) + self._redate_post_images(post) self.session.commit() return True + def _redate_post_images(self, post: Post) -> None: + """Carry a post's date onto the images already linked to it (#4431). + + The native ingesters import a post's media BEFORE its record: the + per-media sidecar holds only the image identity (post-first, #856), and + the date arrives with `_post.json`. So `_attach_provenance` links each + image to a post that has no date yet, and the image keeps its download + time. This runs when the record lands, and applies the same two rules + `_attach_provenance` applies: `effective_date` is the PRIMARY post's + date, and `earliest_post_date` is the earliest date across every post + the image is linked to. Only rows that differ are written.""" + if post.post_date is None: + return + self.session.flush() + self.session.execute( + update(ImageRecord) + .where(ImageRecord.primary_post_id == post.id) + .where(ImageRecord.effective_date.is_distinct_from(post.post_date)) + .values(effective_date=post.post_date) + .execution_options(synchronize_session=False) + ) + linked = select(ImageProvenance.image_record_id).where( + ImageProvenance.post_id == post.id + ) + earliest = ( + select(func.min(Post.post_date)) + .select_from(ImageProvenance) + .join(Post, Post.id == ImageProvenance.post_id) + .where(ImageProvenance.image_record_id == ImageRecord.id) + .where(Post.post_date.is_not(None)) + .correlate(ImageRecord) + .scalar_subquery() + ) + self.session.execute( + update(ImageRecord) + .where(ImageRecord.id.in_(linked)) + .where(ImageRecord.earliest_post_date.is_distinct_from(earliest)) + .values(earliest_post_date=earliest) + .execution_options(synchronize_session=False) + ) + def attach_in_place( self, path: Path, diff --git a/tests/test_redate_images_migration.py b/tests/test_redate_images_migration.py new file mode 100644 index 0000000..8441923 --- /dev/null +++ b/tests/test_redate_images_migration.py @@ -0,0 +1,70 @@ +"""Migration 0113 (#4431): images linked to a post before its date arrived get +the post's date back. Runs the migration's data step against real rows.""" +import importlib.util +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from backend.app.models import Artist, ImageProvenance, ImageRecord, Post +from tests.factories import make_image as _img + +pytestmark = pytest.mark.integration + +_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "alembic" / "versions" / "0113_redate_native_images.py" +) + + +def _redate(): + spec = importlib.util.spec_from_file_location("m0113", _MIGRATION) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.redate_images + + +def _post(db, artist, epid, when): + p = Post(artist_id=artist.id, external_post_id=epid, post_date=when) + db.add(p) + db.flush() + return p + + +def _link(db, img, post, *, primary): + db.add(ImageProvenance(image_record_id=img.id, post_id=post.id)) + if primary: + img.primary_post_id = post.id + db.flush() + + +def test_redate_images_from_their_posts(db_sync): + sent = datetime(2024, 3, 1, 18, 30, tzinfo=UTC) + earlier = datetime(2023, 1, 5, tzinfo=UTC) + artist = Artist(name="Alice", slug="alice") + db_sync.add(artist) + db_sync.flush() + undated = _post(db_sync, artist, "u1", None) + dated = _post(db_sync, artist, "d1", sent) + repost = _post(db_sync, artist, "r1", earlier) + + stale = _img(db_sync, "a" * 64) # primary post dated, image not + reposted = _img(db_sync, "b" * 64) # also in an earlier post + orphan = _img(db_sync, "c" * 64) # only an undated post + _link(db_sync, stale, dated, primary=True) + _link(db_sync, reposted, dated, primary=True) + _link(db_sync, reposted, repost, primary=False) + _link(db_sync, orphan, undated, primary=True) + orphan_before = orphan.effective_date + + _redate()(db_sync.connection()) + db_sync.expire_all() + + stale = db_sync.get(ImageRecord, stale.id) + reposted = db_sync.get(ImageRecord, reposted.id) + orphan = db_sync.get(ImageRecord, orphan.id) + assert stale.effective_date == sent + assert stale.earliest_post_date == sent + assert reposted.effective_date == sent # the primary post's date + assert reposted.earliest_post_date == earlier # the earliest post's date + assert orphan.effective_date == orphan_before # no date to take diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py index 87f1408..6149624 100644 --- a/tests/test_sidecar_import.py +++ b/tests/test_sidecar_import.py @@ -382,3 +382,36 @@ def test_external_links_not_duplicated_on_reimport(importer, import_layout): assert importer.session.execute( select(func.count()).select_from(ExternalLink) ).scalar_one() == 1 + + +def test_post_record_redates_images_linked_before_it(importer, import_layout): + """#4431: the native ingesters import a message's media before its record, + and only the record carries the date. The images start on their download + time; when the record lands they take the post's date.""" + import_root, _ = import_layout + artist = Artist(name="Alice", slug="alice") + importer.session.add(artist) + importer.session.flush() + m = import_root / "Alice" / "20240301_123_01_art.jpg" + _split(m, "v") + _sidecar(m, {"category": "discord", "message_id": "123"}) + r = importer.import_one(m) + assert r.status == "imported" + rec = importer.session.get(ImageRecord, r.image_id) + post = importer.session.execute(select(Post)).scalar_one() + assert post.post_date is None + download_time = rec.effective_date + + sc = import_root / "Alice" / "20240301_123_post.json" + sc.write_text(json.dumps({ + "category": "discord", "message_id": "123", "message": "", + "date": "2024-03-01T18:30:00.000000+00:00", + })) + assert importer.upsert_post_record(sc, artist=artist) is True + importer.session.expire_all() + rec = importer.session.get(ImageRecord, r.image_id) + post = importer.session.execute(select(Post)).scalar_one() + assert post.post_date is not None + assert post.post_date != download_time + assert rec.effective_date == post.post_date + assert rec.earliest_post_date == post.post_date