diff --git a/backend/app/services/post_association_service.py b/backend/app/services/post_association_service.py index 6a80a87..c995363 100644 --- a/backend/app/services/post_association_service.py +++ b/backend/app/services/post_association_service.py @@ -77,7 +77,7 @@ from collections import Counter from dataclasses import dataclass from datetime import UTC, datetime, timedelta -from sqlalchemy import func, or_, select +from sqlalchemy import and_, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from ..models import ImageRecord, ImportSettings, Post, PostAssociation @@ -123,6 +123,12 @@ DECLARED_MENTION = 0.6 MAX_CANDIDATES = 25 +# How many just-grouped drops one sweep will look around. A ceiling on the +# `or_` the sweep builds, not a policy — a backfill that authors thousands of +# drops at once should not turn one sweep into a full-library rescan, which is +# the manual button's job. +MAX_RECENT_DROPS = 200 + @dataclass(frozen=True) class _Corpus: @@ -410,21 +416,60 @@ async def rescan(session: AsyncSession, *, now: datetime | None = None) -> dict: now = now or datetime.now(UTC) window_hours = float(settings.discord_link_window_hours) + window = timedelta(hours=window_hours) # Only look at announcements that could still have a partner in range — # a full-library rescan is the manual button's job, not the sweep's. horizon = now - timedelta(hours=window_hours * 2) sort_key = func.coalesce(Post.post_date, Post.downloaded_at) - ids = (await session.execute( + ids = set((await session.execute( select(Post.id).where( Post.synthesized_by.is_(None), Post.absorbed_by_post_id.is_(None), sort_key >= horizon, ) + )).scalars().all()) + + # ...and announcements sitting next to a drop FC has only JUST authored. + # + # A drop's `post_date` is backdated to its first message, but FC cannot + # write the drop until the message has an embedding and the hourly grouper + # has run — so a drop created this minute can land weeks back in the feed. + # Its neighbours were last swept before it existed, and a sweep keyed only + # on how recent the ANNOUNCEMENT is will never look at them again. + # + # That is #4392's third cause, and it is the one that left a measured 0.800 + # pair with an empty review queue on the live instance. The other two were + # about scoring; this one meant nothing was scored at all. + drop_times = (await session.execute( + select(sort_key).where( + Post.synthesized_by == DROP_GROUPER, + func.coalesce(Post.last_grew_at, Post.downloaded_at) >= horizon, + ) + .order_by(func.coalesce(Post.last_grew_at, Post.downloaded_at).desc()) + .limit(MAX_RECENT_DROPS) )).scalars().all() + # The interval arithmetic is done in Python rather than SQL: a handful of + # literal ranges is portable, and `now - INTERVAL` is not. + ranges = [ + and_(sort_key >= at - window, sort_key <= at + window) + for at in drop_times + if at is not None + ] + if ranges: + ids |= set((await session.execute( + select(Post.id).where( + Post.synthesized_by.is_(None), + Post.absorbed_by_post_id.is_(None), + or_(*ranges), + ) + )).scalars().all()) svc = PostAssociationService(session) proposed = 0 - for pid in ids: + # Sorted because `ids` is now a union of two queries: set iteration order + # is arbitrary, and a sweep that visits posts in a different order each + # run is one whose failures cannot be reproduced. + for pid in sorted(ids): proposed += await svc.match_post( pid, threshold=float(settings.discord_link_threshold), diff --git a/tests/test_post_association.py b/tests/test_post_association.py index a767c8b..15836a4 100644 --- a/tests/test_post_association.py +++ b/tests/test_post_association.py @@ -599,3 +599,65 @@ def test_the_creators_own_phrasing_counts_as_a_declaration(): def test_an_unrelated_server_is_still_not_a_declaration(): """The widened vocabulary must not widen into ordinary prose.""" assert declared_signal("

the servers were down all morning

") == 0.0 + + +@pytest.mark.asyncio +async def test_a_drop_grouped_today_pulls_in_the_post_that_announced_it(db): + """#4392's third cause, and the only one of the three about whether a pair + is SCORED AT ALL rather than how. + + A drop's `post_date` is backdated to its first message, but FC cannot + author the drop until that message has an embedding and the hourly grouper + has run. So a drop created this minute lands wherever its messages were — + here, ten days back, far outside the sweep's horizon. Keyed only on how + recent the announcement is, the sweep looks straight past the pair and + never comes back to it. + + Measured on the live instance: a pair scoring 0.800 with an empty review + queue. + """ + artist, patreon, discord = await _artist_with_channels(db, "lateartist") + now = datetime.now(UTC) + old = now - timedelta(days=10) + teaser = await _teaser( + db, artist, patreon, at=old, body="Full set is on discord.gg/abc", + ) + # post_date is backdated; downloaded_at defaults to now, which is when FC + # actually wrote this row. + drop = await _drop(db, artist, discord, at=old + timedelta(hours=1)) + settings = await ImportSettings.load(db) + settings.discord_link_enabled = True + await db.commit() + + result = await rescan(db) + await db.commit() + + assert result["proposed"] == 1, ( + "the announcement is 10 days old and the drop was authored today — " + "a sweep keyed only on the announcement's own date never sees it" + ) + assoc = (await db.execute(select(PostAssociation))).scalar_one() + assert assoc.announcement_post_id == teaser.id + assert assoc.payload_post_id == drop.id + + +@pytest.mark.asyncio +async def test_the_sweep_still_ignores_a_drop_nothing_is_near(db): + """The widened sweep must not become the full-library rescan it replaced. + A drop authored today with no announcement in range proposes nothing.""" + artist, patreon, discord = await _artist_with_channels(db, "lonelyartist") + now = datetime.now(UTC) + await _teaser( + db, artist, patreon, at=now - timedelta(days=60), + body="Full set is on discord.gg/abc", + ) + await _drop(db, artist, discord, at=now - timedelta(days=10)) + settings = await ImportSettings.load(db) + settings.discord_link_enabled = True + await db.commit() + + result = await rescan(db) + await db.commit() + + assert result["proposed"] == 0 + assert (await db.execute(select(PostAssociation))).scalars().all() == []