fix: a download is marked seen only after it is imported

A run killed between download and import left its files on disk and in the
seen-ledger but never in the library, and every later walk trusted the
ledger. TamadaHeijun's 12PCG post lost 7 of 13 images this way (stranded
run 90402), which read as the duplicates filter.

The ingester now hands phase 3 a mark_seen_after_import hook, called after
the import loop. A file on disk with no ImageRecord at its path is fed to
import, not reconciled into the ledger. Recapture reaches files already
orphaned, because it looks past the ledger to the disk.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

tests/test_download_service.py

@@ -9,9 +9,9 @@ from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
-from sqlalchemy import select
+from sqlalchemy import func, select
 
-from backend.app.models import Artist, DownloadEvent, ImportSettings, Source
+from backend.app.models import Artist, DownloadEvent, ImageRecord, ImportSettings, Source
 from backend.app.services.credential_crypto import CredentialCrypto
 from backend.app.services.credential_service import CredentialService
 from backend.app.services.thumbnailer import Thumbnailer
@@ -152,6 +152,14 @@ async def test_download_source_attaches_written_files(
         files_downloaded=2,
         stdout=f"{f1}\n{f2}\n",
     )
+    # The ledger is marked only once the files are in: a run killed before
+    # import must leave them unmarked so the next walk imports them.
+    imported_when_marked = []
+    result.mark_seen_after_import = lambda: imported_when_marked.append(
+        db_sync.execute(
+            select(func.count(ImageRecord.id)).where(ImageRecord.path.in_([str(f1), str(f2)]))
+        ).scalar_one()
+    )
     fake_gdl = _fake_gdl_with_result(result)
 
     sync_settings = db_sync.execute(
@@ -182,6 +190,7 @@ async def test_download_source_attaches_written_files(
     assert ev.files_count == 2
     assert ev.metadata_["import_summary"]["attached"] == 2
     assert ev.metadata_["run_stats"]["downloaded_count"] == 2
+    assert imported_when_marked == [2]
 
 
 @pytest.mark.asyncio

tests/test_patreon_ingester.py

@@ -7,6 +7,7 @@ real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so
 the tier-1 skip and the idempotent mark-seen run against actual rows.
 """
 
+import hashlib
 from datetime import UTC, datetime, timedelta
 
 import pytest
@@ -15,6 +16,7 @@ from sqlalchemy.orm import sessionmaker
 
 from backend.app.models import (
     Artist,
+    ImageRecord,
     PatreonFailedMedia,
     PatreonSeenMedia,
     Source,
@@ -237,6 +239,9 @@ async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_
     # plan #704: structured run_stats carry the real counts.
     assert result.run_stats["downloaded_count"] == 2
     assert result.posts_processed == 1
+    # The media wait for phase 3 to import them; only the post key is in yet.
+    assert _count_ledger(sync_engine, source_id) == 1
+    result.mark_seen_after_import()
     # 2 media keys + 1 synthetic post key (body/links recaptured per post).
     assert _count_ledger(sync_engine, source_id) == 3
     # The post body + links are captured for media posts too (rides the walk).
@@ -263,6 +268,7 @@ async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_
     assert result.run_stats["quarantined_count"] == 1
     assert result.run_stats["downloaded_count"] == 1
     assert len(result.written_paths) == 1        # quarantined NOT written
+    result.mark_seen_after_import()
     # Quarantined media is NOT marked seen (a fixed file may be re-fetched);
     # m1 + the synthetic post key (body/links captured per post) = 2.
     assert _count_ledger(sync_engine, source_id) == 2
@@ -573,6 +579,7 @@ async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path)
     m1 = _media("p1", 1)
     client = _FakeClient([(None, [("p1", [m1])])])
     # File still on disk (a kept image) → tier-2 spares it even under recovery.
+    _seed_record(sync_engine, tmp_path / "p1_1.jpg")
     downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
     ing = _ingester(sync_engine, tmp_path, client, downloader)
 
@@ -582,6 +589,7 @@ async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path)
     )
     assert result.files_downloaded == 0
     assert downloader.download_calls == 0
+    assert result.written_paths == []
     # Disk-skip reconciles the media key + the synthetic post key (recovery
     # recaptures the body/links per post) = 2.
     assert _count_ledger(sync_engine, source_id) == 2
@@ -609,6 +617,15 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
     assert downloader.post_records == 1
 
 
+def _seed_record(sync_engine, path):
+    """An ImageRecord at `path` — the file was imported, not just downloaded."""
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        s.add(ImageRecord(path=str(path), sha256=hashlib.sha256(str(path).encode()).hexdigest(),
+                          size_bytes=1, mime="image/jpeg", origin="downloaded"))
+        s.commit()
+
+
 def _seed_seen(sync_engine, source_id, key, post_id=None):
     factory = sessionmaker(sync_engine, expire_on_commit=False)
     with factory() as s:
@@ -629,6 +646,7 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
     # Pre-seed BOTH the media key and the synthetic post key as already seen.
     _seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
     _seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
+    _seed_record(sync_engine, tmp_path / "p1_1.jpg")
 
     # 1) Plain backfill: post key is seen → gate skips body recapture.
     client = _FakeClient([(None, [("p1", [m1])])])
@@ -696,6 +714,7 @@ async def test_gated_post_skipped_entirely_no_media_no_record(
     assert downloader.download_calls == 1
     assert len(result.post_record_paths) == 1     # only the open post
     assert downloader.post_records == 1
+    result.mark_seen_after_import()
     # The gated post left NO trace in the seen-ledger (no media key, no post key):
     # only the open post's media key + synthetic post key are recorded.
     assert _count_ledger(sync_engine, source_id) == 2
@@ -827,6 +846,57 @@ async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
     assert result.relink_source_paths == []
 
 
+# --- a run that dies between download and import ---------------------------
+# TamadaHeijun's 【12PCG】 post, 2026-09-24: 13 files on disk, 5 in the library.
+# A run wrote 01–08, marked them seen, and was killed before phase 3 imported
+# them; every later walk trusted the ledger and never looked again.
+
+
+@pytest.mark.asyncio
+async def test_a_run_that_dies_before_import_leaves_its_media_unmarked(
+    source_id, sync_engine, tmp_path,
+):
+    m1 = _media("p1", 1)
+    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]),
+                    _FakeDownloader(tmp_path))
+    ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
+            url="https://patreon.com/ingest", mode="tick")
+    # Phase 3 never ran, so `mark_seen_after_import` never did: only the post key.
+    assert _count_ledger(sync_engine, source_id) == 1
+
+    # The next walk finds the file on disk with no record, and imports it.
+    ing2 = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]),
+                     _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)}))
+    result = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
+                      url="https://patreon.com/ingest", mode="tick")
+    assert result.written_paths == [str(tmp_path / "p1_1.jpg")]
+    assert result.files_downloaded == 0          # not fetched again
+    assert "on disk but never imported: p1_1.jpg" in result.stdout
+    result.mark_seen_after_import()
+    assert _count_ledger(sync_engine, source_id) == 2
+
+
+@pytest.mark.asyncio
+async def test_recapture_imports_a_seen_file_nothing_imported(
+    source_id, sync_engine, tmp_path,
+):
+    """The repair for files already orphaned: they are in the ledger, so only
+    a walk that looks past it — recapture — reaches them."""
+    m1, m2 = _media("p1", 1), _media("p1", 2)
+    for m in (m1, m2):
+        _seed_seen(sync_engine, source_id, _ledger_key(m), post_id="p1")
+    _seed_record(sync_engine, tmp_path / "p1_2.jpg")   # m2 made it in; m1 did not
+    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1), _ledger_key(m2)})
+    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1, m2])])]),
+                    downloader)
+    result = ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
+                     url="https://patreon.com/ingest", mode="recapture")
+
+    assert result.written_paths == [str(tmp_path / "p1_1.jpg")]
+    assert [r[0] for r in result.relink_source_paths] == [str(tmp_path / "p1_2.jpg")]
+    assert downloader.download_calls == 0
+
+
 # --- dead-letter ledger (plan #705 #7) ------------------------------------