fix: a post record is marked seen only after it is upserted, and an hourly sweep dates the posts killed walks left undated (#4436)
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 17s
CI and images / frontend-build (push) Successful in 27s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Failing after 2m25s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 17s
CI and images / frontend-build (push) Successful in 27s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Failing after 2m25s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
ingest_core marked a post's record key seen when it wrote _post.json, but the record reaches the database (and dates the post and its images) only in phase 3. A walk killed before phase 3 left posts the ledger called recorded and the database never dated; ticks early-out long before reaching them again. Keys now join the media in mark_seen_after_import. date_posts_from_records finds each undated native post's record under its artist's folder and upserts it with the post's own source. Hourly on maintenance_long; an empty query once everything is dated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
This commit is contained in:
@@ -415,3 +415,51 @@ def test_post_record_redates_images_linked_before_it(importer, import_layout):
|
||||
assert post.post_date != download_time
|
||||
assert rec.effective_date == post.post_date
|
||||
assert rec.earliest_post_date == post.post_date
|
||||
|
||||
|
||||
def test_an_undated_post_is_dated_from_the_record_its_walk_left_on_disk(
|
||||
importer, import_layout,
|
||||
):
|
||||
"""#4436: a walk killed before phase 3 wrote the message's record to disk
|
||||
but never upserted it, and marked it seen, so no later tick fixed it. The
|
||||
repair finds the record under the artist's folder and dates the post — and
|
||||
its images — under the post's own source."""
|
||||
from backend.app.services.post_record_repair import date_posts_from_records
|
||||
|
||||
import_root, images_root = import_layout
|
||||
artist = Artist(name="Alice", slug="alice")
|
||||
importer.session.add(artist)
|
||||
importer.session.flush()
|
||||
importer.session.add(Source(
|
||||
artist_id=artist.id, platform="discord",
|
||||
url="https://discord.com/channels/1/100",
|
||||
))
|
||||
importer.session.flush()
|
||||
m = import_root / "Alice" / "20240301_123_01_art.jpg"
|
||||
_split(m, "v")
|
||||
_sidecar(m, {"category": "discord", "message_id": "123"})
|
||||
r = importer.import_one(m)
|
||||
assert r.status == "imported"
|
||||
post = importer.session.execute(select(Post)).scalar_one()
|
||||
assert post.post_date is None and post.source_id is not None
|
||||
|
||||
channel = images_root / "alice" / "discord" / "rewards"
|
||||
channel.mkdir(parents=True)
|
||||
(channel / "20240301_123_post.json").write_text(json.dumps({
|
||||
"category": "discord", "message_id": "123", "message": "",
|
||||
"date": "2024-03-01T18:30:00.000000+00:00",
|
||||
}))
|
||||
(channel / "20240301_999_post.json").write_text("not json") # skipped, not fatal
|
||||
|
||||
summary = date_posts_from_records(importer.session, importer, images_root)
|
||||
|
||||
assert summary == {"undated": 1, "dated": 1, "no_record": 0}
|
||||
importer.session.expire_all()
|
||||
post = importer.session.execute(select(Post)).scalar_one()
|
||||
rec = importer.session.get(ImageRecord, r.image_id)
|
||||
assert post.post_date is not None
|
||||
assert rec.earliest_post_date == post.post_date
|
||||
# Nothing left undated: the next sweep is an empty query.
|
||||
assert date_posts_from_records(importer.session, importer, images_root) == {
|
||||
"undated": 0, "dated": 0, "no_record": 0,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user