backend/app/services/post_association_service.py

@@ -77,7 +77,7 @@ from collections import Counter
 from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import func, or_, select
+from sqlalchemy import and_, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..models import ImageRecord, ImportSettings, Post, PostAssociation
@@ -123,6 +123,12 @@ DECLARED_MENTION = 0.6
 
 MAX_CANDIDATES = 25
 
+# How many just-grouped drops one sweep will look around. A ceiling on the
+# `or_` the sweep builds, not a policy — a backfill that authors thousands of
+# drops at once should not turn one sweep into a full-library rescan, which is
+# the manual button's job.
+MAX_RECENT_DROPS = 200
+
 
 @dataclass(frozen=True)
 class _Corpus:
@@ -410,21 +416,60 @@ async def rescan(session: AsyncSession, *, now: datetime | None = None) -> dict:
 
     now = now or datetime.now(UTC)
     window_hours = float(settings.discord_link_window_hours)
+    window = timedelta(hours=window_hours)
     # Only look at announcements that could still have a partner in range —
     # a full-library rescan is the manual button's job, not the sweep's.
     horizon = now - timedelta(hours=window_hours * 2)
     sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
-    ids = (await session.execute(
+    ids = set((await session.execute(
         select(Post.id).where(
             Post.synthesized_by.is_(None),
             Post.absorbed_by_post_id.is_(None),
             sort_key >= horizon,
         )
+    )).scalars().all())
+
+    # ...and announcements sitting next to a drop FC has only JUST authored.
+    #
+    # A drop's `post_date` is backdated to its first message, but FC cannot
+    # write the drop until the message has an embedding and the hourly grouper
+    # has run — so a drop created this minute can land weeks back in the feed.
+    # Its neighbours were last swept before it existed, and a sweep keyed only
+    # on how recent the ANNOUNCEMENT is will never look at them again.
+    #
+    # That is #4392's third cause, and it is the one that left a measured 0.800
+    # pair with an empty review queue on the live instance. The other two were
+    # about scoring; this one meant nothing was scored at all.
+    drop_times = (await session.execute(
+        select(sort_key).where(
+            Post.synthesized_by == DROP_GROUPER,
+            func.coalesce(Post.last_grew_at, Post.downloaded_at) >= horizon,
+        )
+        .order_by(func.coalesce(Post.last_grew_at, Post.downloaded_at).desc())
+        .limit(MAX_RECENT_DROPS)
     )).scalars().all()
+    # The interval arithmetic is done in Python rather than SQL: a handful of
+    # literal ranges is portable, and `now - INTERVAL` is not.
+    ranges = [
+        and_(sort_key >= at - window, sort_key <= at + window)
+        for at in drop_times
+        if at is not None
+    ]
+    if ranges:
+        ids |= set((await session.execute(
+            select(Post.id).where(
+                Post.synthesized_by.is_(None),
+                Post.absorbed_by_post_id.is_(None),
+                or_(*ranges),
+            )
+        )).scalars().all())
 
     svc = PostAssociationService(session)
     proposed = 0
-    for pid in ids:
+    # Sorted because `ids` is now a union of two queries: set iteration order
+    # is arbitrary, and a sweep that visits posts in a different order each
+    # run is one whose failures cannot be reproduced.
+    for pid in sorted(ids):
         proposed += await svc.match_post(
             pid,
             threshold=float(settings.discord_link_threshold),

tests/test_post_association.py

@@ -599,3 +599,65 @@ def test_the_creators_own_phrasing_counts_as_a_declaration():
 def test_an_unrelated_server_is_still_not_a_declaration():
     """The widened vocabulary must not widen into ordinary prose."""
     assert declared_signal("<p>the servers were down all morning</p>") == 0.0
+
+
+@pytest.mark.asyncio
+async def test_a_drop_grouped_today_pulls_in_the_post_that_announced_it(db):
+    """#4392's third cause, and the only one of the three about whether a pair
+    is SCORED AT ALL rather than how.
+
+    A drop's `post_date` is backdated to its first message, but FC cannot
+    author the drop until that message has an embedding and the hourly grouper
+    has run. So a drop created this minute lands wherever its messages were —
+    here, ten days back, far outside the sweep's horizon. Keyed only on how
+    recent the announcement is, the sweep looks straight past the pair and
+    never comes back to it.
+
+    Measured on the live instance: a pair scoring 0.800 with an empty review
+    queue.
+    """
+    artist, patreon, discord = await _artist_with_channels(db, "lateartist")
+    now = datetime.now(UTC)
+    old = now - timedelta(days=10)
+    teaser = await _teaser(
+        db, artist, patreon, at=old, body="Full set is on discord.gg/abc",
+    )
+    # post_date is backdated; downloaded_at defaults to now, which is when FC
+    # actually wrote this row.
+    drop = await _drop(db, artist, discord, at=old + timedelta(hours=1))
+    settings = await ImportSettings.load(db)
+    settings.discord_link_enabled = True
+    await db.commit()
+
+    result = await rescan(db)
+    await db.commit()
+
+    assert result["proposed"] == 1, (
+        "the announcement is 10 days old and the drop was authored today — "
+        "a sweep keyed only on the announcement's own date never sees it"
+    )
+    assoc = (await db.execute(select(PostAssociation))).scalar_one()
+    assert assoc.announcement_post_id == teaser.id
+    assert assoc.payload_post_id == drop.id
+
+
+@pytest.mark.asyncio
+async def test_the_sweep_still_ignores_a_drop_nothing_is_near(db):
+    """The widened sweep must not become the full-library rescan it replaced.
+    A drop authored today with no announcement in range proposes nothing."""
+    artist, patreon, discord = await _artist_with_channels(db, "lonelyartist")
+    now = datetime.now(UTC)
+    await _teaser(
+        db, artist, patreon, at=now - timedelta(days=60),
+        body="Full set is on discord.gg/abc",
+    )
+    await _drop(db, artist, discord, at=now - timedelta(days=10))
+    settings = await ImportSettings.load(db)
+    settings.discord_link_enabled = True
+    await db.commit()
+
+    result = await rescan(db)
+    await db.commit()
+
+    assert result["proposed"] == 0
+    assert (await db.execute(select(PostAssociation))).scalars().all() == []