diff --git a/alembic/versions/0110_unified_post_card.py b/alembic/versions/0110_unified_post_card.py new file mode 100644 index 0000000..f1724fa --- /dev/null +++ b/alembic/versions/0110_unified_post_card.py @@ -0,0 +1,59 @@ +"""The unified post card — fold window, family window, and who linked a pair. + +Milestone 388, #4402 and #4401. A Patreon teaser's card shows the Discord drop +it announced, and the rest of that piece's variants, by REFERENCE: nothing is +absorbed, nothing changes owner, and every Discord post keeps its own place. + +Three columns: + +* `import_settings.discord_link_fold_hours` — a linked drop leaves the feed + only when it is this close to its teaser (the same release, shown twice). +* `import_settings.discord_family_window_days` — how far from the teaser the + card reaches for variants. 60 is measured: named families spread up to 44 + days on artist 8, every collision found over 500. +* `post_association.linked_by` — "fc" or "operator", so a link FC made by + itself can say so on the card and offer the undo the operator asked for. + +Revision ID: 0110 +Revises: 0109 +Create Date: 2026-09-24 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0110" +down_revision = "0109" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "import_settings", + sa.Column( + "discord_link_fold_hours", + sa.Float(), + nullable=False, + server_default=sa.text("24"), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "discord_family_window_days", + sa.Float(), + nullable=False, + server_default=sa.text("60"), + ), + ) + op.add_column( + "post_association", + sa.Column("linked_by", sa.String(length=16), nullable=True), + ) + + +def downgrade(): + op.drop_column("post_association", "linked_by") + op.drop_column("import_settings", "discord_family_window_days") + op.drop_column("import_settings", "discord_link_fold_hours") diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index ffcbd6f..c8bc828 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -45,6 +45,8 @@ _EDITABLE_FIELDS = ( "discord_link_threshold", "discord_link_window_hours", "discord_link_auto", + "discord_link_fold_hours", + "discord_family_window_days", "extdl_mega_enabled", "extdl_gdrive_enabled", "extdl_mediafire_enabled", @@ -182,6 +184,12 @@ async def update_import_settings(): return jsonify( {"error": "discord_link_window_hours must be a positive number"} ), 400 + # Zero is meaningful for both: fold nothing, or reference no variants. + for key in ("discord_link_fold_hours", "discord_family_window_days"): + if key in body: + v = body[key] + if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0: + return jsonify({"error": f"{key} must be a number >= 0"}), 400 if "wip_title_tagging_enabled" in body and not isinstance( body["wip_title_tagging_enabled"], bool ): diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index 35218a6..3a0bba2 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -158,6 +158,31 @@ class ImportSettings(Base): server_default="true", ) + # The unified card (#4402). A linked Discord drop is NOT absorbed into its + # teaser — it keeps its own post, date and provenance, and the teaser's card + # shows it by REFERENCE. Operator, 2026-09-24: *"discord 'posts' land as + # normal and only hidden from the post view they're posted the same day."* + # + # So the drop's own card leaves the feed only when it sits within this many + # hours of the teaser that references it — the adjacency that reads as the + # same thing twice. Hours rather than a calendar day: a teaser at 23:00 and + # its drop at 01:00 are one release, and "the same day" has no timezone + # the server can know. + discord_link_fold_hours: Mapped[float] = mapped_column( + Float, nullable=False, default=24.0, + server_default="24", + ) + # How far from the teaser the card reaches for the rest of a piece's + # variants — the wips, alts and censor passes a creator trickles out under + # one working name (#4401). Measured on artist 8: named families spread a + # median 5 days and up to 44, while every name collision found spreads + # over 500. A reference, not a regrouping, so a generous value costs one + # extra thumbnail at worst — never a post moved or hidden. + discord_family_window_days: Mapped[float] = mapped_column( + Float, nullable=False, default=60.0, + server_default="60", + ) + # #830 off-platform file-host downloads — per-host enable lever (default on, # rule #26). Column names are extdl__enabled so the worker reads them # via getattr(settings, f"extdl_{host}_enabled", True). diff --git a/backend/app/models/post_association.py b/backend/app/models/post_association.py index 96eaa3f..9e78feb 100644 --- a/backend/app/models/post_association.py +++ b/backend/app/models/post_association.py @@ -91,6 +91,13 @@ class PostAssociation(Base): status: Mapped[str] = mapped_column( String(16), nullable=False, server_default="pending", index=True ) + # WHO linked it: "fc" when the matcher linked a conclusive pair by itself + # (discord_link_auto), "operator" when a person accepted it. The card needs + # this to be honest — a link FC asserted on its own says so and offers an + # undo, which the operator chose over a silent merge (#4402). NULL on a row + # that is not linked, and on rows linked before the column existed, all of + # which an operator accepted: auto-linking shipped in the same release. + linked_by: Mapped[str | None] = mapped_column(String(16), nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/services/post_association_service.py b/backend/app/services/post_association_service.py index 737fdc8..f5706c6 100644 --- a/backend/app/services/post_association_service.py +++ b/backend/app/services/post_association_service.py @@ -500,6 +500,7 @@ class PostAssociationService: score=score, signals=signals, status=status, + linked_by="fc" if status == "linked" else None, )) made += 1 return made, linked @@ -526,14 +527,19 @@ class PostAssociationService: if a is None: return None a.status = "linked" + a.linked_by = "operator" return {"id": a.id, "status": a.status} async def dismiss(self, association_id: int) -> dict | None: a = await self.session.get(PostAssociation, association_id) if a is None: return None - # Kept, not deleted — the row is what remembers the rejection. + # Kept, not deleted — the row is what remembers the rejection. It is + # also the undo for a link FC made itself (#4402): the unified card + # dismisses the pair, and the dismissed row stops the next sweep from + # linking it straight back. a.status = "dismissed" + a.linked_by = None return {"id": a.id, "status": a.status} async def linked_for(self, post_ids: list[int]) -> dict[int, list[dict]]: diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index 6207667..e41b5d2 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -19,6 +19,7 @@ from ..models import ( ExternalLink, ImageProvenance, ImageRecord, + ImportSettings, Post, PostAttachment, Source, @@ -118,6 +119,15 @@ class PostFeedService: # from. `around` and `get_post` deliberately do NOT apply this: reaching # a member by id is how you inspect a grouping. stmt = stmt.where(Post.absorbed_by_post_id.is_(None)) + # A linked Discord drop is shown ON its teaser's card by reference + # (#4402), so its own card sitting beside that teaser is the same + # release twice. Only then is it left out — an older drop keeps its + # place, because a reference does not take anything out of history. + fold_hours = await self._fold_hours() + if fold_hours > 0: + from .post_unification import fold_clause + + stmt = stmt.where(fold_clause(fold_hours)) if artist_id is not None: stmt = stmt.where(Post.artist_id == artist_id) if platform is not None: @@ -169,9 +179,12 @@ class PostFeedService: thumbs_map = await self._thumbnails_for(post_ids) atts_map = await self._attachments_for(post_ids) links_map = await self._links_for(post_ids) + unified_map = await self._unified_for([p for p, _, _ in rows]) items = [ - self._to_dict(post, artist, source, thumbs_map, atts_map, links_map) + self._to_dict( + post, artist, source, thumbs_map, atts_map, links_map, unified_map, + ) for post, artist, source in rows ] return {"items": items, "next_cursor": next_cursor} @@ -213,6 +226,7 @@ class PostFeedService: anchor_item = self._to_dict( anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map, await self._links_for([anchor_post.id]), + await self._unified_for([anchor_post]), ) return { "items": newer["items"] + [anchor_item] + older["items"], @@ -239,6 +253,7 @@ class PostFeedService: item = self._to_dict( post, artist, source, thumbs_map, atts_map, await self._links_for([post.id]), + await self._unified_for([post]), ) item["description_full"] = html_to_plain(post.description) # Full (uncapped) translated description for the detail view (#143). @@ -385,9 +400,27 @@ class PostFeedService: return await PostAssociationService(self.session).linked_for(post_ids) + async def _unified_for(self, posts: list[Post]) -> dict[int, dict]: + """The reference set each teaser's card shows (#4402). Local import for + the same reason as `_links_for`: it reaches the association service.""" + from .post_unification import PostUnificationService + + return await PostUnificationService(self.session).unified_for(posts) + + async def _fold_hours(self) -> float: + """`discord_link_fold_hours`, read without assuming the row exists. + + The feed is the one surface that must not fail on a settings row the + caller never needed, so a missing row folds nothing rather than + raising — which is also exactly how the feed behaved before this. + """ + settings = await self.session.get(ImportSettings, 1) + return float(settings.discord_link_fold_hours) if settings is not None else 0.0 + def _to_dict( self, post: Post, artist: Artist, source: Source | None, thumbs_map: dict, atts_map: dict, links_map: dict | None = None, + unified_map: dict | None = None, ) -> dict: plain_full = html_to_plain(post.description) if post.description else None if plain_full is None: @@ -436,6 +469,11 @@ class PostFeedService: # post is the drop). Always a list so the UI never branches on # absence. "associations": (links_map or {}).get(post.id, []), + # #4402. On a teaser with a linked drop: what the card shows BY + # REFERENCE — the drop's images, the piece's older variants, and + # the text of each — plus who made each link, so one FC made by + # itself can say so and offer the undo. None on every other post. + "unified": (unified_map or {}).get(post.id), # Non-null on a chat message a synthetic post absorbed. The feed # filters these out, but `around`/`get_post` still reach them, and # the UI uses this to explain why a post it linked to is not in the diff --git a/backend/app/services/post_naming.py b/backend/app/services/post_naming.py index e63aff4..d20ab52 100644 --- a/backend/app/services/post_naming.py +++ b/backend/app/services/post_naming.py @@ -112,6 +112,10 @@ _STOPWORDS = frozenset({ # `{user[name]}` as it for ~1,600 files, so it is the single most common # "name" in the library and identifies nothing. "none", + # gallery-dl's fallback when a Discord attachment has no filename of its + # own. Measured on artist 8: four unrelated images across 1,974 days, and + # the one false family the leading-name rule admitted inside 60 days. + "image0", }) # A bare year: still needed for the TEXT signal, where words and numbers are @@ -170,19 +174,17 @@ def _strip_prefixes(stem: str) -> str: return _HASH_SUFFIX.sub("", stem) -def working_name_tokens(path: str) -> set[str]: - """The identity-bearing tokens in one image's filename. +def _ordered_tokens(path: str) -> list[str]: + """The identity-bearing tokens of one filename, in the order written. - Returns an EMPTY set for a name that carries no working title — a - screenshot, a bare number, a stopword. Empty means "no evidence", which the - caller must treat as silence rather than as a weak match; see the module - docstring for the false positive that rule exists for. + The one tokenizer both public readings share, so the set of names and the + leading name cannot disagree about what counts as a name. """ stem = _strip_prefixes(PurePosixPath(path).stem) if _SCREENSHOT.match(stem.strip()): - return set() + return [] - out: set[str] = set() + out: list[str] = [] # Hyphens are kept INSIDE tokens — `0-k` is a real working name on the live # instance, and splitting on hyphen would reduce it to a single character # and then discard it for being too short. @@ -192,10 +194,43 @@ def working_name_tokens(path: str) -> set[str]: continue if tok in _STOPWORDS or not _HAS_LETTER.search(tok): continue - out.add(tok) + if tok not in out: + out.append(tok) return out +def working_name_tokens(path: str) -> set[str]: + """The identity-bearing tokens in one image's filename. + + Returns an EMPTY set for a name that carries no working title — a + screenshot, a bare number, a stopword. Empty means "no evidence", which the + caller must treat as silence rather than as a weak match; see the module + docstring for the false positive that rule exists for. + """ + return set(_ordered_tokens(path)) + + +def leading_name(path: str) -> str | None: + """The FIRST identity token of a filename — the piece, not its decoration. + + Creators lead with what the piece is and trail with what this export of it + is: `Year_20k_wip1`, `not_sombra_21-cumpeen`, `Tentacooler_c_ins`. Content + words sit at the tail, and they span too FEW posts for any frequency cap + to catch — `nude`, `cum` and `top` are on three of artist 8's posts each. + Position is what separates them from a name. + + Measured on artist 8, Discord messages 2-60 days apart sharing a gated + token: 121 pairs. The 106 sharing the leading name all read as one piece's + trickle; of the 15 sharing only a trailing word, 14 are sibling pieces + (`Bea_Machamp_Shiny_*` / `Bea_Machoke_Shiny_*`) and one is a plain + collision (`Undyne_insert_bottom_only-C` / `Lichgalclc_Lingerie_Bottom_21`). + + None when the name carries no identity at all — see `working_name_tokens`. + """ + tokens = _ordered_tokens(path) + return tokens[0] if tokens else None + + def token_frequencies(posts: Iterable[Iterable[str]]) -> Counter[str]: """How many of ONE ARTIST's POSTS each working-name token appears in. diff --git a/backend/app/services/post_unification.py b/backend/app/services/post_unification.py new file mode 100644 index 0000000..cc1b902 --- /dev/null +++ b/backend/app/services/post_unification.py @@ -0,0 +1,427 @@ +"""The unified post card — a teaser shows what it points at (#4402, #4401). + +Milestone 388. A Patreon teaser is a POINTER: a cropped, censored fragment +whose job is to say "the full set is in Discord". Until this module the card +rendered the fragment and a text link, and the reader had to make the join FC +had already made. + +Operator, 2026-09-24: *"the teaser from the patreon post doesn't show the items +that it's supposed to reference so I'm trying to unify the teaser post with the +content it's meant to draw attention to."* + +## A reference, never an absorption + +`discord_grouping` folds chat messages into a synthetic post by transferring +ownership (`absorbed_by_post_id`). That is the wrong primitive here, and the +operator said so directly: *"the nested items on the unified post are a +duplicate or reference of existing content. that's why they can show similar +items and not erase or invalidate the way the discord items landed."* + +So nothing here writes. The Discord posts keep their own rows, dates and +places in the feed; the teaser's card DISPLAYS them. That is also what makes +reaching back for older variants safe at all: a wrong reference shows one +extra thumbnail in one place, where a wrong regrouping would move content. + +## What a teaser references + +1. The Discord drops a `linked` PostAssociation joins it to (#4392) — accepted + by the operator, or linked by FC on a conclusive name match. +2. The rest of that piece's VARIANT FAMILY (#4401): the wips, alts and censor + passes a creator trickles out under one working name, days or weeks apart. + +Families are found by the creator's LEADING working name, not by any shared +token and not by image similarity — see `post_naming.leading_name` for the +measurement, and lesson #4400 for why a whole-image comparison between two +works by one artist cannot separate "same piece" from "same artist". +""" +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import datetime, timedelta + +from sqlalchemy import and_, exists, extract, func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import aliased + +from ..models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + PostAssociation, + Source, +) +from ..utils.phash import hamming, hash_bits +from ..utils.text import html_to_plain, truncate_at_word +from .discord_grouping import PLATFORM as DISCORD +from .gallery_service import thumbnail_url +from .post_association_service import DUPLICATE_MAX_DISTANCE +from .post_naming import leading_name, rarity, token_frequencies + +# A leading name spanning this many of ONE ARTIST's posts is a habit — a +# character the creator returns to — not one piece's trickle. +# +# Its own value rather than post_naming.MAX_TOKEN_POSTS (6), which is +# calibrated for PAIRING two posts, and measured too tight for a family. On +# artist 8, `tentacooler` spans 6 posts over 7 days and `0-k1` 6 posts over 10: +# both real families, both gated out at 6. At 8, `anya` (7) passes the cap — +# and has no pair inside the family window, which is what the window is for. +FAMILY_MAX_POSTS = 8 + +# The text each referenced post contributes to the card, per post. The card +# clamps it again; this keeps a long Discord thread from making the feed +# payload the size of the thread. +TEXT_LIMIT = 280 + + +@dataclass(frozen=True) +class Candidate: + """One image as the family search sees it.""" + + image_id: int + post_id: int + path: str + phash: int | None + at: datetime + + +def family( + seed: Iterable[Candidate], + pool: Iterable[Candidate], + name_posts: Counter[str], + hash_posts: Counter[int], + *, + anchor: datetime, + window: timedelta, + max_posts: int = FAMILY_MAX_POSTS, +) -> list[Candidate]: + """The images in `pool` that belong to the same piece as `seed`. + + A member shares a seed image's LEADING working name, or is a perceptual + near-duplicate of one (the same file re-posted), and lies within `window` + of `anchor` — the teaser's date. Oldest first, so the card reads as the + trickle it was. + + ONE hop from the seed, never transitive. Every measured family is one hop + from any of its members, because the members share the name; chaining is + what lets a family drift from `Year_20k` to whatever `Year_20k_Base`'s + other tokens happen to touch. + + Both identity routes are rarity-gated against `max_posts`, exactly as the + matcher gates them. A leading name the creator uses across many posts is a + character, and a hash on many posts is a banner. + """ + seed = list(seed) + names = { + name for c in seed + if (name := leading_name(c.path)) is not None + and rarity(name_posts.get(name, 0), max_posts) > 0 + } + hashes = [ + c.phash for c in seed + if c.phash is not None and rarity(hash_posts.get(c.phash, 0), max_posts) > 0 + ] + taken = {c.image_id for c in seed} + + out: list[Candidate] = [] + for c in pool: + if c.image_id in taken or abs(c.at - anchor) > window: + continue + named = leading_name(c.path) in names + copied = c.phash is not None and any( + (d := hamming(c.phash, h)) is not None and d <= DUPLICATE_MAX_DISTANCE + for h in hashes + ) + if named or copied: + out.append(c) + taken.add(c.image_id) + return sorted(out, key=lambda c: (c.at, c.image_id)) + + +def _when(post: Post) -> datetime: + return post.post_date or post.downloaded_at + + +def _text(post: Post) -> str | None: + plain = html_to_plain(post.description) if post.description else None + if not plain or not plain.strip(): + return None + return truncate_at_word(plain.strip(), TEXT_LIMIT)[0] + + +@dataclass +class _Artist: + """Everything the family search needs about one artist, loaded once.""" + + rows: dict[int, tuple] # image_id -> (post_id, path, phash, sha, mime, thumb) + posts: dict[int, Post] + platform: dict[int, str | None] # post_id -> platform + name_posts: Counter[str] + hash_posts: Counter[int] + + +class PostUnificationService: + def __init__(self, session: AsyncSession): + self.session = session + self._artists: dict[int, _Artist] = {} + + async def _artist(self, artist_id: int) -> _Artist: + if artist_id in self._artists: + return self._artists[artist_id] + + posts: dict[int, Post] = {} + platform: dict[int, str | None] = {} + for post, plat in (await self.session.execute( + select(Post, Source.platform) + .outerjoin(Source, Post.source_id == Source.id) + .where(Post.artist_id == artist_id) + )).all(): + posts[post.id] = post + platform[post.id] = plat + + rows: dict[int, tuple] = {} + paths_by_post: dict[int, list[str]] = {} + hashes_by_post: dict[int, set[int]] = {} + for img_id, post_id, path, phash, sha, mime, thumb in (await self.session.execute( + select( + ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path, + ImageRecord.phash, ImageRecord.sha256, ImageRecord.mime, + ImageRecord.thumbnail_path, + ).where( + ImageRecord.artist_id == artist_id, + ImageRecord.primary_post_id.is_not(None), + ) + )).all(): + bits = hash_bits(phash) + rows[img_id] = (post_id, path, bits, sha, mime, thumb) + paths_by_post.setdefault(post_id, []).append(path) + if bits is not None: + hashes_by_post.setdefault(post_id, set()).add(bits) + + # Counted over EVERY post the artist has, exactly as the matcher's + # corpus counts them — a family is judged against the whole library, + # not against the slice inside the window, or a character name would + # look rare in any quiet month. + found = _Artist( + rows=rows, + posts=posts, + platform=platform, + name_posts=token_frequencies(paths_by_post.values()), + hash_posts=Counter(h for hs in hashes_by_post.values() for h in hs), + ) + self._artists[artist_id] = found + return found + + async def _drop_images(self, drop_ids: list[int]) -> dict[int, list[int]]: + """drop post id -> its image ids, through provenance as the feed reads them. + + A synthetic drop owns no image outright: its images belong to the + member messages, and `discord_grouping` gives the drop a provenance row + for each. The primary_post_id arm keeps any image that has one and no + row, the same union `PostFeedService._thumbnails_for` takes. + """ + out: dict[int, list[int]] = {pid: [] for pid in drop_ids} + if not drop_ids: + return out + links = ( + select( + ImageProvenance.image_record_id.label("image_id"), + ImageProvenance.post_id.label("post_id"), + ) + .where(ImageProvenance.post_id.in_(drop_ids)) + .union( + select( + ImageRecord.id.label("image_id"), + ImageRecord.primary_post_id.label("post_id"), + ).where(ImageRecord.primary_post_id.in_(drop_ids)) + ) + .subquery() + ) + for img_id, pid in (await self.session.execute( + select(links.c.image_id, links.c.post_id).order_by(links.c.image_id) + )).all(): + out[pid].append(img_id) + return out + + async def unified_for(self, posts: Iterable[Post]) -> dict[int, dict]: + """post id -> the card's reference set, for each post that HAS one. + + Only teasers get one: a post with at least one `linked` association on + the announcing side. Every other post is absent from the result, and + the card renders exactly as it did before this module existed. + """ + teasers = {p.id: p for p in posts if p.synthesized_by is None} + if not teasers: + return {} + links = (await self.session.execute( + select(PostAssociation) + .where( + PostAssociation.status == "linked", + PostAssociation.announcement_post_id.in_(list(teasers)), + ) + .order_by(PostAssociation.id) + )).scalars().all() + if not links: + return {} + + settings = await self.session.get(ImportSettings, 1) + window = timedelta(days=float( + settings.discord_family_window_days if settings is not None else 60.0 + )) + drop_images = await self._drop_images( + sorted({a.payload_post_id for a in links}) + ) + + by_teaser: dict[int, list[PostAssociation]] = {} + for a in links: + by_teaser.setdefault(a.announcement_post_id, []).append(a) + + out: dict[int, dict] = {} + for teaser_id, assocs in by_teaser.items(): + teaser = teasers[teaser_id] + artist = await self._artist(teaser.artist_id) + out[teaser_id] = self._compose(teaser, assocs, drop_images, artist, window) + return out + + def _compose( + self, + teaser: Post, + assocs: list[PostAssociation], + drop_images: dict[int, list[int]], + artist: _Artist, + window: timedelta, + ) -> dict: + def candidate(img_id: int) -> Candidate | None: + row = artist.rows.get(img_id) + if row is None: + return None + post_id, path, bits, *_ = row + post = artist.posts.get(post_id) + if post is None: + return None + return Candidate(img_id, post_id, path, bits, _when(post)) + + drop_ids = [a.payload_post_id for a in assocs] + shown = [i for d in drop_ids for i in drop_images.get(d, [])] + own = [i for i, row in artist.rows.items() if row[0] == teaser.id] + seed = [c for i in own + shown if (c := candidate(i)) is not None] + + # Variants come from Discord only. That is where a creator trickles + # them out, it is the corpus the family rule was measured on, and it + # keeps one teaser from pulling a DIFFERENT teaser's crop onto its card. + pool = [ + c for i, row in artist.rows.items() + if artist.platform.get(row[0]) == DISCORD + and (c := candidate(i)) is not None + ] + variants = family( + seed, pool, artist.name_posts, artist.hash_posts, + anchor=_when(teaser), window=window, + ) + + def thumb(img_id: int, post_id: int, role: str) -> dict | None: + row = artist.rows.get(img_id) + if row is None: + return None + _pid, _path, _bits, sha, mime, tp = row + return { + "image_id": img_id, + "thumbnail_url": thumbnail_url(tp, sha, mime), + "mime": mime, + "post_id": post_id, + "role": role, + } + + own_ids = set(own) + thumbnails: list[dict] = [] + seen: set[int] = set(own_ids) + for drop_id in drop_ids: + for img_id in drop_images.get(drop_id, []): + if img_id in seen: + continue + if (t := thumb(img_id, drop_id, "drop")) is not None: + thumbnails.append(t) + seen.add(img_id) + for c in variants: + if c.image_id in seen: + continue + if (t := thumb(c.image_id, c.post_id, "variant")) is not None: + thumbnails.append(t) + seen.add(c.image_id) + + # The text of every item the card unifies — the operator's *"the + # unified card should also contain the text for any of the items + # unified on it"*. A drop's own description already joins its member + # messages, so a variant's text is its MESSAGE, read off the member + # post that owns the image. A line said twice (`@everyone 🍈🍈` on + # every message of a drop) is shown once. + texts: list[dict] = [] + said: set[str] = set() + + def add_text(post: Post | None, role: str) -> None: + if post is None: + return + text = _text(post) + if text is None or text in said: + return + said.add(text) + texts.append({ + "post_id": post.id, + "role": role, + "date": _when(post).isoformat(), + "text": text, + }) + + for drop_id in drop_ids: + add_text(artist.posts.get(drop_id), "drop") + for post_id in dict.fromkeys(c.post_id for c in variants): + add_text(artist.posts.get(post_id), "variant") + + return { + "links": [ + { + "association_id": a.id, + "post_id": a.payload_post_id, + # "fc" | "operator" | None (linked before the column + # existed — an operator accept, every one of them). + "linked_by": a.linked_by, + "token": (a.signals or {}).get("identity_token"), + } + for a in assocs + ], + "thumbnails": thumbnails, + "variant_count": sum(1 for t in thumbnails if t["role"] == "variant"), + "texts": texts, + } + + +def fold_clause(fold_hours: float): + """WHERE clause: this post is NOT a linked drop sitting beside its teaser. + + Operator: *"discord 'posts' land as normal and only hidden from the post + view they're posted the same day."* Everything else stays — an older + variant the teaser also references is history, and a reference does not + remove it from history. + + Built on `post_date`/`downloaded_at`, not the feed's `resurfaced_at`-led + sort key: whether two posts are the same release is a question about when + they were published, not about where the feed has since moved one. + """ + teaser = aliased(Post) + # The SQL-standard EXTRACT(epoch FROM …), which every Postgres accepts. + gap = func.abs(extract( + "epoch", + func.coalesce(Post.post_date, Post.downloaded_at) + - func.coalesce(teaser.post_date, teaser.downloaded_at), + )) + return ~exists( + select(PostAssociation.id) + .join(teaser, teaser.id == PostAssociation.announcement_post_id) + .where(and_( + PostAssociation.payload_post_id == Post.id, + PostAssociation.status == "linked", + gap <= fold_hours * 3600, + )) + ) diff --git a/tests/test_api_settings_downloader.py b/tests/test_api_settings_downloader.py index dbaebde..093ca6c 100644 --- a/tests/test_api_settings_downloader.py +++ b/tests/test_api_settings_downloader.py @@ -135,3 +135,34 @@ async def test_auto_linking_refuses_a_non_boolean(client): "/api/settings/import", json={"discord_link_auto": "yes"} ) assert resp.status_code == 400 + + +# --- #4402: the unified card's two windows --------------------------------- + + +@pytest.mark.asyncio +async def test_the_unified_card_windows_ship_with_their_measured_defaults(client): + """24h is "the same day" without a timezone the server cannot know; 60 days + is the measured family boundary (named families up to 44d on artist 8, + every collision over 500).""" + resp = await client.get("/api/settings/import") + body = await resp.get_json() + assert body["discord_link_fold_hours"] == 24 + assert body["discord_family_window_days"] == 60 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key", ["discord_link_fold_hours", "discord_family_window_days"]) +async def test_zero_turns_a_unified_card_window_off(client, key): + """Zero is a real answer for both: fold nothing, reference no variants.""" + resp = await client.patch("/api/settings/import", json={key: 0}) + assert resp.status_code == 200 + assert (await resp.get_json())[key] == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key", ["discord_link_fold_hours", "discord_family_window_days"]) +@pytest.mark.parametrize("bad", [-1, "24", True]) +async def test_a_unified_card_window_refuses_nonsense(client, key, bad): + resp = await client.patch("/api/settings/import", json={key: bad}) + assert resp.status_code == 400 diff --git a/tests/test_post_naming.py b/tests/test_post_naming.py index 3bc241b..0b2e2ed 100644 --- a/tests/test_post_naming.py +++ b/tests/test_post_naming.py @@ -20,6 +20,7 @@ from backend.app.services.post_naming import ( IDENTITY_FLOOR, MAX_MARKER_POSTS, MAX_TOKEN_POSTS, + leading_name, marker_frequencies, marker_overlap, shared_identity, @@ -305,3 +306,47 @@ def test_marker_overlap_cannot_be_called_without_the_frequencies(): write by accident. A default would have kept it one keyword away.""" with pytest.raises(TypeError): marker_overlap("\U0001F348", "\U0001F348") + + +# --- the leading name: what a variant family is keyed on (#4401) ------------ + + +@pytest.mark.parametrize( + "path, expected", + [ + ("20240301_1213141516171819_01_Year_20K_wip3.png", "year"), + ("20240414_1213141516171820_01_not_sombra_21-cumpeen.png", "not"), + ("20240101_1213141516171821_02_Tentacooler_c_ins.png", "tentacooler"), + ("01_((0-k.jpg", "0-k"), + # The legacy era's `0071` has no letter, so the name is the first + # token that IS one — not "whatever came first". + ("85317841_media_212565911_0071 NoHeart__c3118a69f3__c3118a69f3.jpg", "noheart"), + ], +) +def test_the_leading_name_is_the_piece_not_its_decoration(path, expected): + assert leading_name(path) == expected + + +def test_a_trailing_content_word_is_never_the_leading_name(): + """Measured: `Undyne_insert_bottom_only-C` and `Lichgalclc_Lingerie_Bottom_21` + share `bottom`, 56 days apart, and are unrelated. `bottom` spans only three + posts, so no frequency cap can refuse it — position is what does.""" + assert leading_name("Undyne_insert_bottom_only-C.png") == "undyne" + assert leading_name("Lichgalclc_Lingerie_Bottom_21.png") == "lichgalclc" + + +@pytest.mark.parametrize( + "path", + ["01_Screenshot 2026-08-13 000004.png", "20240101_1213141516171822_01_image0.png"], +) +def test_a_name_with_no_identity_has_no_leading_name(path): + """`image0` is gallery-dl's fallback for an attachment with no name — + measured on four unrelated images across 1,974 days.""" + assert leading_name(path) is None + + +def test_the_leading_name_is_always_one_of_the_names(): + """One tokenizer serves both readings, so they cannot disagree about what + counts as a name.""" + path = "20240301_1213141516171819_01_Year_20K_wip3.png" + assert leading_name(path) in working_name_tokens(path) diff --git a/tests/test_post_unification.py b/tests/test_post_unification.py new file mode 100644 index 0000000..618c1ec --- /dev/null +++ b/tests/test_post_unification.py @@ -0,0 +1,395 @@ +"""#4402 / #4401: a teaser's card shows what it points at — by reference. + +Operator, 2026-09-24: *"the teaser from the patreon post doesn't show the items +that it's supposed to reference"*, and on how: *"discord 'posts' land as normal +and only hidden from the post view they're posted the same day. the nested +items on the unified post are a duplicate or reference of existing content."* + +Two properties carry the whole design, and most of what follows pins them: + +* NOTHING MOVES. A referenced Discord post keeps its row, its date and its + place in the feed; only a drop sitting beside its own teaser is left out. +* A family is the creator's LEADING working name, one hop from the seed, inside + a window. The refusals are the measured false positives, not invented ones. +""" +from collections import Counter +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import select + +from backend.app.models import ( + Artist, + ImageRecord, + ImportSettings, + Post, + PostAssociation, + Source, +) +from backend.app.services.discord_grouping import DROP_GROUPER +from backend.app.services.post_association_service import PostAssociationService +from backend.app.services.post_feed_service import PostFeedService +from backend.app.services.post_unification import ( + FAMILY_MAX_POSTS, + Candidate, + family, +) + +T0 = datetime(2026, 9, 1, 12, 0, tzinfo=UTC) +WINDOW = timedelta(days=60) + + +def _c(image_id, name, *, days=0, post_id=None, phash=None): + """One image, Discord-shaped, `days` from T0.""" + return Candidate( + image_id=image_id, + post_id=post_id if post_id is not None else image_id, + path=f"20260901_12345678901234{image_id:04d}_01_{name}.png", + phash=phash, + at=T0 + timedelta(days=days), + ) + + +def _family(seed, pool, *, names=None, hashes=None): + return family( + seed, pool, Counter(names or {}), Counter(hashes or {}), + anchor=T0, window=WINDOW, + ) + + +# --- the family rule, pure --------------------------------------------------- + + +def test_the_older_wips_of_the_same_piece_are_its_family(): + """The operator's ask exactly: *"yellowroom trickles out variants and I want + them to show in the grouped post even if they're older"*. Measured shape: + `Year_20k_wip1 -> wip3 -> Base -> Cndm` over 44 days.""" + seed = [_c(1, "Year_20k_Cndm")] + pool = [_c(2, "Year_20k_wip1", days=-44), _c(3, "Year_20K_wip3", days=-44), + _c(4, "Year_20k_Base", days=-20)] + + assert [c.image_id for c in _family(seed, pool)] == [2, 3, 4] + + +def test_a_family_reads_oldest_first(): + """So the card shows the trickle in the order it happened.""" + seed = [_c(1, "svtt_drench_b")] + pool = [_c(2, "svtt_wip5", days=-1), _c(3, "svtt_wip1", days=-3)] + + assert [c.image_id for c in _family(seed, pool)] == [3, 2] + + +def test_a_namesake_outside_the_window_is_not_family(): + """Time does most of the work. Measured on artist 8: every collision found + spreads over 500 days — `ashley` 1258, `anya` 1217, `image0` 1974.""" + seed = [_c(1, "Ashley_TAIGA")] + pool = [_c(2, "Ashley_Re4_A", days=-1258)] + + assert _family(seed, pool) == [] + + +def test_a_shared_trailing_word_is_not_family(): + """The one plain collision inside 60 days on artist 8: `bottom`, three + posts, 56 days apart. No frequency cap can refuse a word that rare — the + leading-name rule does.""" + seed = [_c(1, "Undyne_insert_bottom_only-C")] + pool = [_c(2, "Lichgalclc_Lingerie_Bottom_21", days=-56)] + + assert _family(seed, pool) == [] + + +def test_a_leading_name_the_creator_uses_everywhere_is_not_a_family(): + """A character is a habit, not a piece. At the cap the name is gated even + inside the window.""" + seed = [_c(1, "Bea_Machamp_Shiny")] + pool = [_c(2, "Bea_Machoke_Shiny", days=-5)] + + assert _family(seed, pool, names={"bea": FAMILY_MAX_POSTS}) == [] + + +def test_the_family_cap_admits_the_measured_long_families(): + """`tentacooler` spans 6 posts over 7 days and `0-k1` 6 over 10 — real + families, both lost at the pairing cap of 6. That is why the family cap is + its own number.""" + seed = [_c(1, "Tentacooler")] + pool = [_c(2, "Tentacooler_c_ins", days=-7)] + + assert [c.image_id for c in _family(seed, pool, names={"tentacooler": 6})] == [2] + + +def test_a_near_duplicate_joins_the_family_without_a_name(): + """Half of this creator's teasers are screenshots, which carry no name. The + same file re-posted is still the same file.""" + seed = [_c(1, "image0", phash=0b1011)] + pool = [_c(2, "image0", days=-10, phash=0b1010)] + + assert [c.image_id for c in _family(seed, pool)] == [2] + + +def test_a_distant_hash_is_not_a_duplicate(): + """Lesson #4400: same-artist images are similar whether or not they are the + same piece, so only a NEAR-duplicate counts — the matcher's own line.""" + seed = [_c(1, "image0", phash=0)] + pool = [_c(2, "image0", days=-10, phash=(1 << 100) - 1)] + + assert _family(seed, pool) == [] + + +def test_a_family_is_one_hop_from_the_seed(): + """Chaining is what lets a family drift: `alpha` reaches an image that also + carries `beta`, and `beta` must not then reach its own family.""" + seed = [_c(1, "alpha_final")] + pool = [_c(2, "alpha_beta", days=-3), _c(3, "beta_wip1", days=-4)] + + assert [c.image_id for c in _family(seed, pool)] == [2] + + +def test_the_seed_never_comes_back_as_its_own_family(): + seed = [_c(1, "Year_20k_Cndm")] + + assert _family(seed, seed + [_c(2, "Year_20k_Base", days=-20)])[0].image_id == 2 + assert len(_family(seed, seed)) == 0 + + +# --- the card, end to end ---------------------------------------------------- + + +async def _channels(db, name): + artist = Artist(name=name, slug=name) + db.add(artist) + await db.flush() + patreon = Source(artist_id=artist.id, platform="patreon", + url=f"https://patreon.com/{name}", enabled=True) + discord = Source(artist_id=artist.id, platform="discord", + url=f"https://discord.com/channels/1/{name}", enabled=True) + db.add_all([patreon, discord]) + await db.flush() + return artist, patreon, discord + + +_seq = iter(range(1, 10_000)) + + +async def _post(db, artist, source, *, at, names, body=None, title=None, synthetic=False): + post = Post( + source_id=source.id, artist_id=artist.id, + external_post_id=f"ext-{next(_seq)}", post_date=at, + post_title=title, description=body, + synthesized_by=DROP_GROUPER if synthetic else None, + ) + db.add(post) + await db.flush() + for name in names: + n = next(_seq) + db.add(ImageRecord( + # Discord-shaped whatever the platform: the prefix is stripped, so + # the NAME below is the leading name. + path=f"/images/{artist.slug}/20260901_1234567890{n:06d}_01_{name}.png", + sha256=f"{n:064d}", size_bytes=10, mime="image/png", + width=10, height=10, origin="downloaded", + primary_post_id=post.id, artist_id=artist.id, + )) + await db.flush() + return post + + +async def _link(db, teaser, drop, *, status="linked", linked_by="fc"): + a = PostAssociation( + announcement_post_id=teaser.id, payload_post_id=drop.id, score=1.0, + signals={"identity": 1.0, "identity_token": "year"}, + status=status, linked_by=linked_by if status == "linked" else None, + ) + db.add(a) + await db.flush() + return a + + +async def _feed(db, artist): + page = await PostFeedService(db).scroll(artist_id=artist.id, limit=100) + return {item["id"]: item for item in page["items"]} + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_a_teaser_shows_the_drop_it_announced(db): + """The complaint itself: the card now carries the drop's images and text.""" + artist, patreon, discord = await _channels(db, "unifyartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"], + title="Year 20k", body="Full set in the server") + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1), + names=["Year_20k_Cndm", "Year_20k_Cndm_alt"], + body="@everyone 🍈🍈 the full set", synthetic=True) + assoc = await _link(db, teaser, drop) + await db.commit() + + unified = (await _feed(db, artist))[teaser.id]["unified"] + + assert [t["role"] for t in unified["thumbnails"]] == ["drop", "drop"] + assert {t["post_id"] for t in unified["thumbnails"]} == {drop.id} + assert unified["links"] == [{ + "association_id": assoc.id, "post_id": drop.id, + "linked_by": "fc", "token": "year", + }] + assert unified["texts"][0]["text"] == "@everyone 🍈🍈 the full set" + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_a_drop_beside_its_teaser_leaves_the_feed(db): + """*"only hidden from the post view they're posted the same day"*.""" + artist, patreon, discord = await _channels(db, "foldartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=2), + names=["Year_20k_Cndm"], synthetic=True) + await _link(db, teaser, drop) + await db.commit() + + feed = await _feed(db, artist) + + assert teaser.id in feed + assert drop.id not in feed + # Left out of the FEED, not out of existence: reachable by id, as every + # post is, because it is still the images' true origin. + assert (await PostFeedService(db).get_post(drop.id))["id"] == drop.id + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_a_drop_days_from_its_teaser_keeps_its_place(db): + """A reference does not take anything out of history.""" + artist, patreon, discord = await _channels(db, "keepartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(days=3), + names=["Year_20k_Cndm"], synthetic=True) + await _link(db, teaser, drop) + await db.commit() + + feed = await _feed(db, artist) + + assert drop.id in feed + assert feed[teaser.id]["unified"]["thumbnails"][0]["post_id"] == drop.id + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_a_proposal_changes_nothing_on_the_card(db): + """Only a LINKED pair unifies. A pending proposal is a question for the + review queue, and rendering it would assert a link nobody made.""" + artist, patreon, discord = await _channels(db, "pendingartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1), + names=["Year_20k_Cndm"], synthetic=True) + await _link(db, teaser, drop, status="pending") + await db.commit() + + feed = await _feed(db, artist) + + assert feed[teaser.id]["unified"] is None + assert drop.id in feed + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_the_older_variants_come_along_and_nothing_else(db): + """The family reaches back 44 days for the wips — and not 90 days for a + namesake, and not at all for a message that only shares a trailing word.""" + artist, patreon, discord = await _channels(db, "familyartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1), + names=["Year_20k_Cndm"], synthetic=True) + wip = await _post(db, artist, discord, at=T0 - timedelta(days=44), + names=["Year_20k_wip1"], body="wip, feedback welcome") + base = await _post(db, artist, discord, at=T0 - timedelta(days=20), + names=["Year_20K_Base"]) + too_old = await _post(db, artist, discord, at=T0 - timedelta(days=90), + names=["Year_20k_old"]) + trailing = await _post(db, artist, discord, at=T0 - timedelta(days=5), + names=["Other_piece_year"]) + await _link(db, teaser, drop) + await db.commit() + + feed = await _feed(db, artist) + unified = feed[teaser.id]["unified"] + variants = [t["post_id"] for t in unified["thumbnails"] if t["role"] == "variant"] + + assert variants == [wip.id, base.id] + assert unified["variant_count"] == 2 + assert too_old.id not in variants and trailing.id not in variants + assert "wip, feedback welcome" in [t["text"] for t in unified["texts"]] + # Referenced, not moved: every variant is still in the feed on its own. + assert {wip.id, base.id} <= set(feed) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_the_family_window_is_the_operators_setting(db): + artist, patreon, discord = await _channels(db, "windowartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1), + names=["Year_20k_Cndm"], synthetic=True) + await _post(db, artist, discord, at=T0 - timedelta(days=20), names=["Year_20k_wip1"]) + await _link(db, teaser, drop) + settings = await db.get(ImportSettings, 1) + settings.discord_family_window_days = 10 + await db.commit() + + unified = (await _feed(db, artist))[teaser.id]["unified"] + + assert unified["variant_count"] == 0 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_undo_is_a_dismissal_and_everything_returns(db): + """The operator chose *"nest automatically, with visible undo"*. The undo + is the review queue's own dismiss: the drop comes back to the feed, the + card loses its references, and the dismissed row is what stops the next + sweep linking the pair straight back.""" + artist, patreon, discord = await _channels(db, "undoartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1), + names=["Year_20k_Cndm"], synthetic=True) + assoc = await _link(db, teaser, drop) + await db.commit() + + await PostAssociationService(db).dismiss(assoc.id) + await db.commit() + feed = await _feed(db, artist) + + assert feed[teaser.id]["unified"] is None + assert drop.id in feed + row = (await db.execute(select(PostAssociation))).scalar_one() + assert (row.status, row.linked_by) == ("dismissed", None) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_an_operator_accept_is_recorded_as_theirs(db): + """So the card does not claim FC made a link a person made.""" + artist, patreon, discord = await _channels(db, "acceptartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1), + names=["Year_20k_Cndm"], synthetic=True) + assoc = await _link(db, teaser, drop, status="pending") + await db.commit() + + await PostAssociationService(db).accept(assoc.id) + await db.commit() + + unified = (await _feed(db, artist))[teaser.id]["unified"] + assert unified["links"][0]["linked_by"] == "operator" + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_a_fold_window_of_zero_hides_nothing(db): + artist, patreon, discord = await _channels(db, "nofoldartist") + teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"]) + drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1), + names=["Year_20k_Cndm"], synthetic=True) + await _link(db, teaser, drop) + settings = await db.get(ImportSettings, 1) + settings.discord_link_fold_hours = 0 + await db.commit() + + assert drop.id in await _feed(db, artist)