fix: a Discord message that re-posts a file is grouped, and joins the drop it repeats

Grouping read a message's images through primary_post_id alone, which only
the first message imported with a file holds. The backfill runs newest-first,
so the original message usually owned nothing: 101 of Yellowroom's messages
(mostly 2018-2020) could never be grouped. Every image they carried also sat
in another message (296 links, measured on the live instance).

_message_images unions ownership with provenance for the candidate query,
the drop seed, the member image links, and the merge pass. Nothing is
re-owned. A later drop carrying the very file an earlier one carries now
merges by a new same_image route, which nearest-neighbour could not see
because it skips the image itself.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

backend/app/services/discord_grouping.py

@@ -63,7 +63,7 @@ from collections import Counter
 from dataclasses import dataclass, field
 from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import Select, delete, func, select, update
+from sqlalchemy import Select, delete, func, select, union, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -126,6 +126,31 @@ def cosine_distance(a, b) -> float:
     return 1.0 - (dot / (na * nb))
 
 
+def _message_images(posts):
+    """(post_id, image_id) for every image a message carries — owned AND re-posted.
+
+    A message owns an image through `primary_post_id`, but only the FIRST
+    message imported with a given file does. The same file posted again is a
+    provenance link, and the backfill runs newest-first, so it is usually the
+    ORIGINAL message that ends up owning nothing. Reading ownership alone left
+    101 of Yellowroom's messages (2018–2020 mostly) ungroupable: every image
+    they carried also sat in another message.
+
+    `posts` is a list of ids or a select of them; filtering both branches by it
+    keeps the union to the messages in hand rather than the whole library.
+    Callers pass MESSAGE posts only — a drop's own provenance rows would read
+    as images it carries.
+    """
+    owned = select(
+        ImageRecord.primary_post_id.label("post_id"), ImageRecord.id.label("image_id"),
+    ).where(ImageRecord.primary_post_id.in_(posts))
+    reposted = select(
+        ImageProvenance.post_id.label("post_id"),
+        ImageProvenance.image_record_id.label("image_id"),
+    ).where(ImageProvenance.post_id.in_(posts))
+    return union(owned, reposted).subquery()
+
+
 def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
     """Ungrouped Discord message-posts, one representative image each, OLDEST
     FIRST — which is the order `build_groups` requires.
@@ -143,13 +168,15 @@ def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
     take the OLDEST candidates instead of the lowest-numbered ones.
     """
     sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
+    carried = _message_images(select(Post.id).where(Post.source_id == source_id))
     inner = (
         select(
             Post.id.label("post_id"),
             sort_key.label("occurred_at"),
             ImageRecord.siglip_embedding.label("embedding"),
         )
-        .join(ImageRecord, ImageRecord.primary_post_id == Post.id)
+        .join(carried, carried.c.post_id == Post.id)
+        .join(ImageRecord, ImageRecord.id == carried.c.image_id)
         .where(
             Post.source_id == source_id,
             # Never absorb a post FC wrote, and never re-absorb one already
@@ -295,9 +322,10 @@ async def _link_member_images(
     """
     if not member_ids:
         return 0
-    image_rows = (await session.execute(
-        select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids))
-    )).scalars().all()
+    carried = _message_images(member_ids)
+    image_rows = sorted(set((await session.execute(
+        select(carried.c.image_id)
+    )).scalars().all()))
     if not image_rows:
         return 0
     await session.execute(
@@ -421,9 +449,11 @@ async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None
     free to disagree; this way there is one.
     """
     sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
+    carried = _message_images(select(Post.id).where(Post.absorbed_by_post_id == post_id))
     return (await session.execute(
         select(ImageRecord.siglip_embedding)
-        .join(Post, ImageRecord.primary_post_id == Post.id)
+        .join(carried, carried.c.image_id == ImageRecord.id)
+        .join(Post, Post.id == carried.c.post_id)
         .where(
             Post.absorbed_by_post_id == post_id,
             ImageRecord.siglip_embedding.is_not(None),
@@ -725,12 +755,15 @@ async def _load_drops(session: AsyncSession, source: Source) -> list[_Drop]:
     names: dict[int, set[str]] = {pid: set() for pid in by_id}
     images: dict[int, list[int]] = {pid: [] for pid in by_id}
     if owner_of:
-        for iid, primary, path in (await session.execute(
-            select(ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path)
-            .where(ImageRecord.primary_post_id.in_(list(owner_of)))
+        carried = _message_images(list(owner_of))
+        for iid, message, path in (await session.execute(
+            select(ImageRecord.id, carried.c.post_id, ImageRecord.path)
+            .join(carried, carried.c.image_id == ImageRecord.id)
             .order_by(ImageRecord.id)
         )).all():
-            drop = owner_of[primary]
+            drop = owner_of[message]
+            if iid in images[drop]:
+                continue  # one file carried by two of the drop's messages
             images[drop].append(iid)
             if (name := leading_name(path)) is not None:
                 names[drop].add(name)
@@ -844,7 +877,12 @@ async def merge_trickles(
             if drop.first_at - earlier.last_at > gap:
                 continue
             shared = mine & gated(earlier.names)
-            if shared:
+            if set(drop.images) & set(earlier.images):
+                # The creator posted the very same file again — the strongest
+                # evidence there is, and one nearest-neighbour cannot see: it
+                # skips the image itself, which is the one they share.
+                targets[earlier.post.id] = (earlier, "same_image")
+            elif shared:
                 targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
             elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
                 targets[earlier.post.id] = (earlier, "nearest")