fix: a post record is marked seen only after it is upserted, and an hourly sweep dates the posts killed walks left undated (#4436)

ingest_core marked a post's record key seen when it wrote _post.json, but
the record reaches the database (and dates the post and its images) only
in phase 3. A walk killed before phase 3 left posts the ledger called
recorded and the database never dated; ticks early-out long before
reaching them again. Keys now join the media in mark_seen_after_import.

date_posts_from_records finds each undated native post's record under
its artist's folder and upserts it with the post's own source. Hourly on
maintenance_long; an empty query once everything is dated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

tests/test_celery_routing.py

@@ -28,6 +28,7 @@ def test_quick_maintenance_stays_on_maintenance():
     ("backend.app.tasks.translation.translate_posts", "maintenance_long"),
     ("backend.app.tasks.gpu_queue.enqueue_gpu_backfill", "maintenance"),
     ("backend.app.tasks.maintenance.backfill_phash", "maintenance_long"),
+    ("backend.app.tasks.maintenance.date_posts_from_records", "maintenance_long"),
     ("backend.app.tasks.admin.normalize_tags_task", "maintenance_long"),
     ("backend.app.tasks.backup.backup_db_task", "maintenance_long"),
     ("backend.app.tasks.maintenance.vacuum_analyze", "maintenance"),

tests/test_patreon_ingester.py

@@ -943,8 +943,10 @@ async def test_a_run_that_dies_before_import_leaves_its_media_unmarked(
                     _FakeDownloader(tmp_path))
     ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
             url="https://patreon.com/ingest", mode="tick")
-    # Phase 3 never ran, so `mark_seen_after_import` never did: only the post key.
-    assert _count_ledger(sync_engine, source_id) == 1
+    # Phase 3 never ran, so `mark_seen_after_import` never did: neither the
+    # media nor the post record is marked (#4436 — the record, marked at write
+    # time, left the post undated for good once the run died before phase 3).
+    assert _count_ledger(sync_engine, source_id) == 0
 
     # The next walk finds the file on disk with no record, and imports it.
     ing2 = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]),
@@ -954,6 +956,7 @@ async def test_a_run_that_dies_before_import_leaves_its_media_unmarked(
     assert result.written_paths == [str(tmp_path / "p1_1.jpg")]
     assert result.files_downloaded == 0          # not fetched again
     assert "on disk but never imported: p1_1.jpg" in result.stdout
+    assert len(result.post_record_paths) == 1    # the record is written again
     result.mark_seen_after_import()
     assert _count_ledger(sync_engine, source_id) == 2
 

tests/test_sidecar_import.py

@@ -415,3 +415,51 @@ def test_post_record_redates_images_linked_before_it(importer, import_layout):
     assert post.post_date != download_time
     assert rec.effective_date == post.post_date
     assert rec.earliest_post_date == post.post_date
+
+
+def test_an_undated_post_is_dated_from_the_record_its_walk_left_on_disk(
+    importer, import_layout,
+):
+    """#4436: a walk killed before phase 3 wrote the message's record to disk
+    but never upserted it, and marked it seen, so no later tick fixed it. The
+    repair finds the record under the artist's folder and dates the post — and
+    its images — under the post's own source."""
+    from backend.app.services.post_record_repair import date_posts_from_records
+
+    import_root, images_root = import_layout
+    artist = Artist(name="Alice", slug="alice")
+    importer.session.add(artist)
+    importer.session.flush()
+    importer.session.add(Source(
+        artist_id=artist.id, platform="discord",
+        url="https://discord.com/channels/1/100",
+    ))
+    importer.session.flush()
+    m = import_root / "Alice" / "20240301_123_01_art.jpg"
+    _split(m, "v")
+    _sidecar(m, {"category": "discord", "message_id": "123"})
+    r = importer.import_one(m)
+    assert r.status == "imported"
+    post = importer.session.execute(select(Post)).scalar_one()
+    assert post.post_date is None and post.source_id is not None
+
+    channel = images_root / "alice" / "discord" / "rewards"
+    channel.mkdir(parents=True)
+    (channel / "20240301_123_post.json").write_text(json.dumps({
+        "category": "discord", "message_id": "123", "message": "",
+        "date": "2024-03-01T18:30:00.000000+00:00",
+    }))
+    (channel / "20240301_999_post.json").write_text("not json")  # skipped, not fatal
+
+    summary = date_posts_from_records(importer.session, importer, images_root)
+
+    assert summary == {"undated": 1, "dated": 1, "no_record": 0}
+    importer.session.expire_all()
+    post = importer.session.execute(select(Post)).scalar_one()
+    rec = importer.session.get(ImageRecord, r.image_id)
+    assert post.post_date is not None
+    assert rec.earliest_post_date == post.post_date
+    # Nothing left undated: the next sweep is an empty query.
+    assert date_posts_from_records(importer.session, importer, images_root) == {
+        "undated": 0, "dated": 0, "no_record": 0,
+    }