feat: a teaser's card references the drop it announced and the piece's variants (4402, 4401)

The operator's problem: a Patreon teaser is a pointer, and its card showed the
censored crop plus a text link while the content it pointed at sat on another
card. The fix is a REFERENCE, not an absorption: "the nested items on the
unified post are a duplicate or reference of existing content". Nothing is
written. Discord posts keep their own rows, dates and places in the feed.

- post_unification: for each teaser with a linked association, the drop's
  images and text, plus its variant family: Discord images sharing the
  seed's gated LEADING working name, or a phash near-duplicate, within a
  window of the teaser. One hop only, oldest first.
- Measured on artist 8 before writing it: of 121 message pairs 2-60 days
  apart that share a gated token, 106 share the leading name and all read
  as real families. Of the 15 sharing only a trailing word, 14 are sibling
  pieces and one is a plain collision (`bottom`, 56 days). The family cap
  is 8, not the pairing cap of 6, because `tentacooler` and `0-k1` (6 posts
  each) are real families.
- The feed drops a linked drop's own card only within discord_link_fold_hours
  of its teaser (default 24): "only hidden from the post view they're posted
  the same day". Older referenced posts stay where they landed.
- post_association.linked_by records whether FC or a person made the link,
  so the card can say so. Undo is the existing dismiss.
- `image0` (gallery-dl's fallback name) becomes a stopword.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
This commit is contained in:
2026-09-24 11:25:52 -04:00
co-authored by Claude Opus 5.5
parent f7b3e15014
commit 30a263a47a
11 changed files with 1087 additions and 11 deletions
@@ -0,0 +1,59 @@
"""The unified post card — fold window, family window, and who linked a pair.
Milestone 388, #4402 and #4401. A Patreon teaser's card shows the Discord drop
it announced, and the rest of that piece's variants, by REFERENCE: nothing is
absorbed, nothing changes owner, and every Discord post keeps its own place.
Three columns:
* `import_settings.discord_link_fold_hours` — a linked drop leaves the feed
only when it is this close to its teaser (the same release, shown twice).
* `import_settings.discord_family_window_days` — how far from the teaser the
card reaches for variants. 60 is measured: named families spread up to 44
days on artist 8, every collision found over 500.
* `post_association.linked_by` — "fc" or "operator", so a link FC made by
itself can say so on the card and offer the undo the operator asked for.
Revision ID: 0110
Revises: 0109
Create Date: 2026-09-24
"""
import sqlalchemy as sa
from alembic import op
revision = "0110"
down_revision = "0109"
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
"import_settings",
sa.Column(
"discord_link_fold_hours",
sa.Float(),
nullable=False,
server_default=sa.text("24"),
),
)
op.add_column(
"import_settings",
sa.Column(
"discord_family_window_days",
sa.Float(),
nullable=False,
server_default=sa.text("60"),
),
)
op.add_column(
"post_association",
sa.Column("linked_by", sa.String(length=16), nullable=True),
)
def downgrade():
op.drop_column("post_association", "linked_by")
op.drop_column("import_settings", "discord_family_window_days")
op.drop_column("import_settings", "discord_link_fold_hours")
+8
View File
@@ -45,6 +45,8 @@ _EDITABLE_FIELDS = (
"discord_link_threshold",
"discord_link_window_hours",
"discord_link_auto",
"discord_link_fold_hours",
"discord_family_window_days",
"extdl_mega_enabled",
"extdl_gdrive_enabled",
"extdl_mediafire_enabled",
@@ -182,6 +184,12 @@ async def update_import_settings():
return jsonify(
{"error": "discord_link_window_hours must be a positive number"}
), 400
# Zero is meaningful for both: fold nothing, or reference no variants.
for key in ("discord_link_fold_hours", "discord_family_window_days"):
if key in body:
v = body[key]
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0:
return jsonify({"error": f"{key} must be a number >= 0"}), 400
if "wip_title_tagging_enabled" in body and not isinstance(
body["wip_title_tagging_enabled"], bool
):
+25
View File
@@ -158,6 +158,31 @@ class ImportSettings(Base):
server_default="true",
)
# The unified card (#4402). A linked Discord drop is NOT absorbed into its
# teaser — it keeps its own post, date and provenance, and the teaser's card
# shows it by REFERENCE. Operator, 2026-09-24: *"discord 'posts' land as
# normal and only hidden from the post view they're posted the same day."*
#
# So the drop's own card leaves the feed only when it sits within this many
# hours of the teaser that references it — the adjacency that reads as the
# same thing twice. Hours rather than a calendar day: a teaser at 23:00 and
# its drop at 01:00 are one release, and "the same day" has no timezone
# the server can know.
discord_link_fold_hours: Mapped[float] = mapped_column(
Float, nullable=False, default=24.0,
server_default="24",
)
# How far from the teaser the card reaches for the rest of a piece's
# variants — the wips, alts and censor passes a creator trickles out under
# one working name (#4401). Measured on artist 8: named families spread a
# median 5 days and up to 44, while every name collision found spreads
# over 500. A reference, not a regrouping, so a generous value costs one
# extra thumbnail at worst — never a post moved or hidden.
discord_family_window_days: Mapped[float] = mapped_column(
Float, nullable=False, default=60.0,
server_default="60",
)
# #830 off-platform file-host downloads — per-host enable lever (default on,
# rule #26). Column names are extdl_<host>_enabled so the worker reads them
# via getattr(settings, f"extdl_{host}_enabled", True).
+7
View File
@@ -91,6 +91,13 @@ class PostAssociation(Base):
status: Mapped[str] = mapped_column(
String(16), nullable=False, server_default="pending", index=True
)
# WHO linked it: "fc" when the matcher linked a conclusive pair by itself
# (discord_link_auto), "operator" when a person accepted it. The card needs
# this to be honest — a link FC asserted on its own says so and offers an
# undo, which the operator chose over a silent merge (#4402). NULL on a row
# that is not linked, and on rows linked before the column existed, all of
# which an operator accepted: auto-linking shipped in the same release.
linked_by: Mapped[str | None] = mapped_column(String(16), nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
@@ -500,6 +500,7 @@ class PostAssociationService:
score=score,
signals=signals,
status=status,
linked_by="fc" if status == "linked" else None,
))
made += 1
return made, linked
@@ -526,14 +527,19 @@ class PostAssociationService:
if a is None:
return None
a.status = "linked"
a.linked_by = "operator"
return {"id": a.id, "status": a.status}
async def dismiss(self, association_id: int) -> dict | None:
a = await self.session.get(PostAssociation, association_id)
if a is None:
return None
# Kept, not deleted — the row is what remembers the rejection.
# Kept, not deleted — the row is what remembers the rejection. It is
# also the undo for a link FC made itself (#4402): the unified card
# dismisses the pair, and the dismissed row stops the next sweep from
# linking it straight back.
a.status = "dismissed"
a.linked_by = None
return {"id": a.id, "status": a.status}
async def linked_for(self, post_ids: list[int]) -> dict[int, list[dict]]:
+39 -1
View File
@@ -19,6 +19,7 @@ from ..models import (
ExternalLink,
ImageProvenance,
ImageRecord,
ImportSettings,
Post,
PostAttachment,
Source,
@@ -118,6 +119,15 @@ class PostFeedService:
# from. `around` and `get_post` deliberately do NOT apply this: reaching
# a member by id is how you inspect a grouping.
stmt = stmt.where(Post.absorbed_by_post_id.is_(None))
# A linked Discord drop is shown ON its teaser's card by reference
# (#4402), so its own card sitting beside that teaser is the same
# release twice. Only then is it left out — an older drop keeps its
# place, because a reference does not take anything out of history.
fold_hours = await self._fold_hours()
if fold_hours > 0:
from .post_unification import fold_clause
stmt = stmt.where(fold_clause(fold_hours))
if artist_id is not None:
stmt = stmt.where(Post.artist_id == artist_id)
if platform is not None:
@@ -169,9 +179,12 @@ class PostFeedService:
thumbs_map = await self._thumbnails_for(post_ids)
atts_map = await self._attachments_for(post_ids)
links_map = await self._links_for(post_ids)
unified_map = await self._unified_for([p for p, _, _ in rows])
items = [
self._to_dict(post, artist, source, thumbs_map, atts_map, links_map)
self._to_dict(
post, artist, source, thumbs_map, atts_map, links_map, unified_map,
)
for post, artist, source in rows
]
return {"items": items, "next_cursor": next_cursor}
@@ -213,6 +226,7 @@ class PostFeedService:
anchor_item = self._to_dict(
anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map,
await self._links_for([anchor_post.id]),
await self._unified_for([anchor_post]),
)
return {
"items": newer["items"] + [anchor_item] + older["items"],
@@ -239,6 +253,7 @@ class PostFeedService:
item = self._to_dict(
post, artist, source, thumbs_map, atts_map,
await self._links_for([post.id]),
await self._unified_for([post]),
)
item["description_full"] = html_to_plain(post.description)
# Full (uncapped) translated description for the detail view (#143).
@@ -385,9 +400,27 @@ class PostFeedService:
return await PostAssociationService(self.session).linked_for(post_ids)
async def _unified_for(self, posts: list[Post]) -> dict[int, dict]:
"""The reference set each teaser's card shows (#4402). Local import for
the same reason as `_links_for`: it reaches the association service."""
from .post_unification import PostUnificationService
return await PostUnificationService(self.session).unified_for(posts)
async def _fold_hours(self) -> float:
"""`discord_link_fold_hours`, read without assuming the row exists.
The feed is the one surface that must not fail on a settings row the
caller never needed, so a missing row folds nothing rather than
raising — which is also exactly how the feed behaved before this.
"""
settings = await self.session.get(ImportSettings, 1)
return float(settings.discord_link_fold_hours) if settings is not None else 0.0
def _to_dict(
self, post: Post, artist: Artist, source: Source | None,
thumbs_map: dict, atts_map: dict, links_map: dict | None = None,
unified_map: dict | None = None,
) -> dict:
plain_full = html_to_plain(post.description) if post.description else None
if plain_full is None:
@@ -436,6 +469,11 @@ class PostFeedService:
# post is the drop). Always a list so the UI never branches on
# absence.
"associations": (links_map or {}).get(post.id, []),
# #4402. On a teaser with a linked drop: what the card shows BY
# REFERENCE — the drop's images, the piece's older variants, and
# the text of each — plus who made each link, so one FC made by
# itself can say so and offer the undo. None on every other post.
"unified": (unified_map or {}).get(post.id),
# Non-null on a chat message a synthetic post absorbed. The feed
# filters these out, but `around`/`get_post` still reach them, and
# the UI uses this to explain why a post it linked to is not in the
+44 -9
View File
@@ -112,6 +112,10 @@ _STOPWORDS = frozenset({
# `{user[name]}` as it for ~1,600 files, so it is the single most common
# "name" in the library and identifies nothing.
"none",
# gallery-dl's fallback when a Discord attachment has no filename of its
# own. Measured on artist 8: four unrelated images across 1,974 days, and
# the one false family the leading-name rule admitted inside 60 days.
"image0",
})
# A bare year: still needed for the TEXT signal, where words and numbers are
@@ -170,19 +174,17 @@ def _strip_prefixes(stem: str) -> str:
return _HASH_SUFFIX.sub("", stem)
def working_name_tokens(path: str) -> set[str]:
"""The identity-bearing tokens in one image's filename.
def _ordered_tokens(path: str) -> list[str]:
"""The identity-bearing tokens of one filename, in the order written.
Returns an EMPTY set for a name that carries no working title — a
screenshot, a bare number, a stopword. Empty means "no evidence", which the
caller must treat as silence rather than as a weak match; see the module
docstring for the false positive that rule exists for.
The one tokenizer both public readings share, so the set of names and the
leading name cannot disagree about what counts as a name.
"""
stem = _strip_prefixes(PurePosixPath(path).stem)
if _SCREENSHOT.match(stem.strip()):
return set()
return []
out: set[str] = set()
out: list[str] = []
# Hyphens are kept INSIDE tokens — `0-k` is a real working name on the live
# instance, and splitting on hyphen would reduce it to a single character
# and then discard it for being too short.
@@ -192,10 +194,43 @@ def working_name_tokens(path: str) -> set[str]:
continue
if tok in _STOPWORDS or not _HAS_LETTER.search(tok):
continue
out.add(tok)
if tok not in out:
out.append(tok)
return out
def working_name_tokens(path: str) -> set[str]:
"""The identity-bearing tokens in one image's filename.
Returns an EMPTY set for a name that carries no working title — a
screenshot, a bare number, a stopword. Empty means "no evidence", which the
caller must treat as silence rather than as a weak match; see the module
docstring for the false positive that rule exists for.
"""
return set(_ordered_tokens(path))
def leading_name(path: str) -> str | None:
"""The FIRST identity token of a filename — the piece, not its decoration.
Creators lead with what the piece is and trail with what this export of it
is: `Year_20k_wip1`, `not_sombra_21-cumpeen`, `Tentacooler_c_ins`. Content
words sit at the tail, and they span too FEW posts for any frequency cap
to catch — `nude`, `cum` and `top` are on three of artist 8's posts each.
Position is what separates them from a name.
Measured on artist 8, Discord messages 2-60 days apart sharing a gated
token: 121 pairs. The 106 sharing the leading name all read as one piece's
trickle; of the 15 sharing only a trailing word, 14 are sibling pieces
(`Bea_Machamp_Shiny_*` / `Bea_Machoke_Shiny_*`) and one is a plain
collision (`Undyne_insert_bottom_only-C` / `Lichgalclc_Lingerie_Bottom_21`).
None when the name carries no identity at all — see `working_name_tokens`.
"""
tokens = _ordered_tokens(path)
return tokens[0] if tokens else None
def token_frequencies(posts: Iterable[Iterable[str]]) -> Counter[str]:
"""How many of ONE ARTIST's POSTS each working-name token appears in.
+427
View File
@@ -0,0 +1,427 @@
"""The unified post card — a teaser shows what it points at (#4402, #4401).
Milestone 388. A Patreon teaser is a POINTER: a cropped, censored fragment
whose job is to say "the full set is in Discord". Until this module the card
rendered the fragment and a text link, and the reader had to make the join FC
had already made.
Operator, 2026-09-24: *"the teaser from the patreon post doesn't show the items
that it's supposed to reference so I'm trying to unify the teaser post with the
content it's meant to draw attention to."*
## A reference, never an absorption
`discord_grouping` folds chat messages into a synthetic post by transferring
ownership (`absorbed_by_post_id`). That is the wrong primitive here, and the
operator said so directly: *"the nested items on the unified post are a
duplicate or reference of existing content. that's why they can show similar
items and not erase or invalidate the way the discord items landed."*
So nothing here writes. The Discord posts keep their own rows, dates and
places in the feed; the teaser's card DISPLAYS them. That is also what makes
reaching back for older variants safe at all: a wrong reference shows one
extra thumbnail in one place, where a wrong regrouping would move content.
## What a teaser references
1. The Discord drops a `linked` PostAssociation joins it to (#4392) — accepted
by the operator, or linked by FC on a conclusive name match.
2. The rest of that piece's VARIANT FAMILY (#4401): the wips, alts and censor
passes a creator trickles out under one working name, days or weeks apart.
Families are found by the creator's LEADING working name, not by any shared
token and not by image similarity — see `post_naming.leading_name` for the
measurement, and lesson #4400 for why a whole-image comparison between two
works by one artist cannot separate "same piece" from "same artist".
"""
from __future__ import annotations
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass
from datetime import datetime, timedelta
from sqlalchemy import and_, exists, extract, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import aliased
from ..models import (
ImageProvenance,
ImageRecord,
ImportSettings,
Post,
PostAssociation,
Source,
)
from ..utils.phash import hamming, hash_bits
from ..utils.text import html_to_plain, truncate_at_word
from .discord_grouping import PLATFORM as DISCORD
from .gallery_service import thumbnail_url
from .post_association_service import DUPLICATE_MAX_DISTANCE
from .post_naming import leading_name, rarity, token_frequencies
# A leading name spanning this many of ONE ARTIST's posts is a habit — a
# character the creator returns to — not one piece's trickle.
#
# Its own value rather than post_naming.MAX_TOKEN_POSTS (6), which is
# calibrated for PAIRING two posts, and measured too tight for a family. On
# artist 8, `tentacooler` spans 6 posts over 7 days and `0-k1` 6 posts over 10:
# both real families, both gated out at 6. At 8, `anya` (7) passes the cap —
# and has no pair inside the family window, which is what the window is for.
FAMILY_MAX_POSTS = 8
# The text each referenced post contributes to the card, per post. The card
# clamps it again; this keeps a long Discord thread from making the feed
# payload the size of the thread.
TEXT_LIMIT = 280
@dataclass(frozen=True)
class Candidate:
"""One image as the family search sees it."""
image_id: int
post_id: int
path: str
phash: int | None
at: datetime
def family(
seed: Iterable[Candidate],
pool: Iterable[Candidate],
name_posts: Counter[str],
hash_posts: Counter[int],
*,
anchor: datetime,
window: timedelta,
max_posts: int = FAMILY_MAX_POSTS,
) -> list[Candidate]:
"""The images in `pool` that belong to the same piece as `seed`.
A member shares a seed image's LEADING working name, or is a perceptual
near-duplicate of one (the same file re-posted), and lies within `window`
of `anchor` — the teaser's date. Oldest first, so the card reads as the
trickle it was.
ONE hop from the seed, never transitive. Every measured family is one hop
from any of its members, because the members share the name; chaining is
what lets a family drift from `Year_20k` to whatever `Year_20k_Base`'s
other tokens happen to touch.
Both identity routes are rarity-gated against `max_posts`, exactly as the
matcher gates them. A leading name the creator uses across many posts is a
character, and a hash on many posts is a banner.
"""
seed = list(seed)
names = {
name for c in seed
if (name := leading_name(c.path)) is not None
and rarity(name_posts.get(name, 0), max_posts) > 0
}
hashes = [
c.phash for c in seed
if c.phash is not None and rarity(hash_posts.get(c.phash, 0), max_posts) > 0
]
taken = {c.image_id for c in seed}
out: list[Candidate] = []
for c in pool:
if c.image_id in taken or abs(c.at - anchor) > window:
continue
named = leading_name(c.path) in names
copied = c.phash is not None and any(
(d := hamming(c.phash, h)) is not None and d <= DUPLICATE_MAX_DISTANCE
for h in hashes
)
if named or copied:
out.append(c)
taken.add(c.image_id)
return sorted(out, key=lambda c: (c.at, c.image_id))
def _when(post: Post) -> datetime:
return post.post_date or post.downloaded_at
def _text(post: Post) -> str | None:
plain = html_to_plain(post.description) if post.description else None
if not plain or not plain.strip():
return None
return truncate_at_word(plain.strip(), TEXT_LIMIT)[0]
@dataclass
class _Artist:
"""Everything the family search needs about one artist, loaded once."""
rows: dict[int, tuple] # image_id -> (post_id, path, phash, sha, mime, thumb)
posts: dict[int, Post]
platform: dict[int, str | None] # post_id -> platform
name_posts: Counter[str]
hash_posts: Counter[int]
class PostUnificationService:
def __init__(self, session: AsyncSession):
self.session = session
self._artists: dict[int, _Artist] = {}
async def _artist(self, artist_id: int) -> _Artist:
if artist_id in self._artists:
return self._artists[artist_id]
posts: dict[int, Post] = {}
platform: dict[int, str | None] = {}
for post, plat in (await self.session.execute(
select(Post, Source.platform)
.outerjoin(Source, Post.source_id == Source.id)
.where(Post.artist_id == artist_id)
)).all():
posts[post.id] = post
platform[post.id] = plat
rows: dict[int, tuple] = {}
paths_by_post: dict[int, list[str]] = {}
hashes_by_post: dict[int, set[int]] = {}
for img_id, post_id, path, phash, sha, mime, thumb in (await self.session.execute(
select(
ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path,
ImageRecord.phash, ImageRecord.sha256, ImageRecord.mime,
ImageRecord.thumbnail_path,
).where(
ImageRecord.artist_id == artist_id,
ImageRecord.primary_post_id.is_not(None),
)
)).all():
bits = hash_bits(phash)
rows[img_id] = (post_id, path, bits, sha, mime, thumb)
paths_by_post.setdefault(post_id, []).append(path)
if bits is not None:
hashes_by_post.setdefault(post_id, set()).add(bits)
# Counted over EVERY post the artist has, exactly as the matcher's
# corpus counts them — a family is judged against the whole library,
# not against the slice inside the window, or a character name would
# look rare in any quiet month.
found = _Artist(
rows=rows,
posts=posts,
platform=platform,
name_posts=token_frequencies(paths_by_post.values()),
hash_posts=Counter(h for hs in hashes_by_post.values() for h in hs),
)
self._artists[artist_id] = found
return found
async def _drop_images(self, drop_ids: list[int]) -> dict[int, list[int]]:
"""drop post id -> its image ids, through provenance as the feed reads them.
A synthetic drop owns no image outright: its images belong to the
member messages, and `discord_grouping` gives the drop a provenance row
for each. The primary_post_id arm keeps any image that has one and no
row, the same union `PostFeedService._thumbnails_for` takes.
"""
out: dict[int, list[int]] = {pid: [] for pid in drop_ids}
if not drop_ids:
return out
links = (
select(
ImageProvenance.image_record_id.label("image_id"),
ImageProvenance.post_id.label("post_id"),
)
.where(ImageProvenance.post_id.in_(drop_ids))
.union(
select(
ImageRecord.id.label("image_id"),
ImageRecord.primary_post_id.label("post_id"),
).where(ImageRecord.primary_post_id.in_(drop_ids))
)
.subquery()
)
for img_id, pid in (await self.session.execute(
select(links.c.image_id, links.c.post_id).order_by(links.c.image_id)
)).all():
out[pid].append(img_id)
return out
async def unified_for(self, posts: Iterable[Post]) -> dict[int, dict]:
"""post id -> the card's reference set, for each post that HAS one.
Only teasers get one: a post with at least one `linked` association on
the announcing side. Every other post is absent from the result, and
the card renders exactly as it did before this module existed.
"""
teasers = {p.id: p for p in posts if p.synthesized_by is None}
if not teasers:
return {}
links = (await self.session.execute(
select(PostAssociation)
.where(
PostAssociation.status == "linked",
PostAssociation.announcement_post_id.in_(list(teasers)),
)
.order_by(PostAssociation.id)
)).scalars().all()
if not links:
return {}
settings = await self.session.get(ImportSettings, 1)
window = timedelta(days=float(
settings.discord_family_window_days if settings is not None else 60.0
))
drop_images = await self._drop_images(
sorted({a.payload_post_id for a in links})
)
by_teaser: dict[int, list[PostAssociation]] = {}
for a in links:
by_teaser.setdefault(a.announcement_post_id, []).append(a)
out: dict[int, dict] = {}
for teaser_id, assocs in by_teaser.items():
teaser = teasers[teaser_id]
artist = await self._artist(teaser.artist_id)
out[teaser_id] = self._compose(teaser, assocs, drop_images, artist, window)
return out
def _compose(
self,
teaser: Post,
assocs: list[PostAssociation],
drop_images: dict[int, list[int]],
artist: _Artist,
window: timedelta,
) -> dict:
def candidate(img_id: int) -> Candidate | None:
row = artist.rows.get(img_id)
if row is None:
return None
post_id, path, bits, *_ = row
post = artist.posts.get(post_id)
if post is None:
return None
return Candidate(img_id, post_id, path, bits, _when(post))
drop_ids = [a.payload_post_id for a in assocs]
shown = [i for d in drop_ids for i in drop_images.get(d, [])]
own = [i for i, row in artist.rows.items() if row[0] == teaser.id]
seed = [c for i in own + shown if (c := candidate(i)) is not None]
# Variants come from Discord only. That is where a creator trickles
# them out, it is the corpus the family rule was measured on, and it
# keeps one teaser from pulling a DIFFERENT teaser's crop onto its card.
pool = [
c for i, row in artist.rows.items()
if artist.platform.get(row[0]) == DISCORD
and (c := candidate(i)) is not None
]
variants = family(
seed, pool, artist.name_posts, artist.hash_posts,
anchor=_when(teaser), window=window,
)
def thumb(img_id: int, post_id: int, role: str) -> dict | None:
row = artist.rows.get(img_id)
if row is None:
return None
_pid, _path, _bits, sha, mime, tp = row
return {
"image_id": img_id,
"thumbnail_url": thumbnail_url(tp, sha, mime),
"mime": mime,
"post_id": post_id,
"role": role,
}
own_ids = set(own)
thumbnails: list[dict] = []
seen: set[int] = set(own_ids)
for drop_id in drop_ids:
for img_id in drop_images.get(drop_id, []):
if img_id in seen:
continue
if (t := thumb(img_id, drop_id, "drop")) is not None:
thumbnails.append(t)
seen.add(img_id)
for c in variants:
if c.image_id in seen:
continue
if (t := thumb(c.image_id, c.post_id, "variant")) is not None:
thumbnails.append(t)
seen.add(c.image_id)
# The text of every item the card unifies — the operator's *"the
# unified card should also contain the text for any of the items
# unified on it"*. A drop's own description already joins its member
# messages, so a variant's text is its MESSAGE, read off the member
# post that owns the image. A line said twice (`@everyone 🍈🍈` on
# every message of a drop) is shown once.
texts: list[dict] = []
said: set[str] = set()
def add_text(post: Post | None, role: str) -> None:
if post is None:
return
text = _text(post)
if text is None or text in said:
return
said.add(text)
texts.append({
"post_id": post.id,
"role": role,
"date": _when(post).isoformat(),
"text": text,
})
for drop_id in drop_ids:
add_text(artist.posts.get(drop_id), "drop")
for post_id in dict.fromkeys(c.post_id for c in variants):
add_text(artist.posts.get(post_id), "variant")
return {
"links": [
{
"association_id": a.id,
"post_id": a.payload_post_id,
# "fc" | "operator" | None (linked before the column
# existed — an operator accept, every one of them).
"linked_by": a.linked_by,
"token": (a.signals or {}).get("identity_token"),
}
for a in assocs
],
"thumbnails": thumbnails,
"variant_count": sum(1 for t in thumbnails if t["role"] == "variant"),
"texts": texts,
}
def fold_clause(fold_hours: float):
"""WHERE clause: this post is NOT a linked drop sitting beside its teaser.
Operator: *"discord 'posts' land as normal and only hidden from the post
view they're posted the same day."* Everything else stays — an older
variant the teaser also references is history, and a reference does not
remove it from history.
Built on `post_date`/`downloaded_at`, not the feed's `resurfaced_at`-led
sort key: whether two posts are the same release is a question about when
they were published, not about where the feed has since moved one.
"""
teaser = aliased(Post)
# The SQL-standard EXTRACT(epoch FROM …), which every Postgres accepts.
gap = func.abs(extract(
"epoch",
func.coalesce(Post.post_date, Post.downloaded_at)
- func.coalesce(teaser.post_date, teaser.downloaded_at),
))
return ~exists(
select(PostAssociation.id)
.join(teaser, teaser.id == PostAssociation.announcement_post_id)
.where(and_(
PostAssociation.payload_post_id == Post.id,
PostAssociation.status == "linked",
gap <= fold_hours * 3600,
))
)
+31
View File
@@ -135,3 +135,34 @@ async def test_auto_linking_refuses_a_non_boolean(client):
"/api/settings/import", json={"discord_link_auto": "yes"}
)
assert resp.status_code == 400
# --- #4402: the unified card's two windows ---------------------------------
@pytest.mark.asyncio
async def test_the_unified_card_windows_ship_with_their_measured_defaults(client):
"""24h is "the same day" without a timezone the server cannot know; 60 days
is the measured family boundary (named families up to 44d on artist 8,
every collision over 500)."""
resp = await client.get("/api/settings/import")
body = await resp.get_json()
assert body["discord_link_fold_hours"] == 24
assert body["discord_family_window_days"] == 60
@pytest.mark.asyncio
@pytest.mark.parametrize("key", ["discord_link_fold_hours", "discord_family_window_days"])
async def test_zero_turns_a_unified_card_window_off(client, key):
"""Zero is a real answer for both: fold nothing, reference no variants."""
resp = await client.patch("/api/settings/import", json={key: 0})
assert resp.status_code == 200
assert (await resp.get_json())[key] == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("key", ["discord_link_fold_hours", "discord_family_window_days"])
@pytest.mark.parametrize("bad", [-1, "24", True])
async def test_a_unified_card_window_refuses_nonsense(client, key, bad):
resp = await client.patch("/api/settings/import", json={key: bad})
assert resp.status_code == 400
+45
View File
@@ -20,6 +20,7 @@ from backend.app.services.post_naming import (
IDENTITY_FLOOR,
MAX_MARKER_POSTS,
MAX_TOKEN_POSTS,
leading_name,
marker_frequencies,
marker_overlap,
shared_identity,
@@ -305,3 +306,47 @@ def test_marker_overlap_cannot_be_called_without_the_frequencies():
write by accident. A default would have kept it one keyword away."""
with pytest.raises(TypeError):
marker_overlap("\U0001F348", "\U0001F348")
# --- the leading name: what a variant family is keyed on (#4401) ------------
@pytest.mark.parametrize(
"path, expected",
[
("20240301_1213141516171819_01_Year_20K_wip3.png", "year"),
("20240414_1213141516171820_01_not_sombra_21-cumpeen.png", "not"),
("20240101_1213141516171821_02_Tentacooler_c_ins.png", "tentacooler"),
("01_((0-k.jpg", "0-k"),
# The legacy era's `0071` has no letter, so the name is the first
# token that IS one — not "whatever came first".
("85317841_media_212565911_0071 NoHeart__c3118a69f3__c3118a69f3.jpg", "noheart"),
],
)
def test_the_leading_name_is_the_piece_not_its_decoration(path, expected):
assert leading_name(path) == expected
def test_a_trailing_content_word_is_never_the_leading_name():
"""Measured: `Undyne_insert_bottom_only-C` and `Lichgalclc_Lingerie_Bottom_21`
share `bottom`, 56 days apart, and are unrelated. `bottom` spans only three
posts, so no frequency cap can refuse it — position is what does."""
assert leading_name("Undyne_insert_bottom_only-C.png") == "undyne"
assert leading_name("Lichgalclc_Lingerie_Bottom_21.png") == "lichgalclc"
@pytest.mark.parametrize(
"path",
["01_Screenshot 2026-08-13 000004.png", "20240101_1213141516171822_01_image0.png"],
)
def test_a_name_with_no_identity_has_no_leading_name(path):
"""`image0` is gallery-dl's fallback for an attachment with no name —
measured on four unrelated images across 1,974 days."""
assert leading_name(path) is None
def test_the_leading_name_is_always_one_of_the_names():
"""One tokenizer serves both readings, so they cannot disagree about what
counts as a name."""
path = "20240301_1213141516171819_01_Year_20K_wip3.png"
assert leading_name(path) in working_name_tokens(path)
+395
View File
@@ -0,0 +1,395 @@
"""#4402 / #4401: a teaser's card shows what it points at — by reference.
Operator, 2026-09-24: *"the teaser from the patreon post doesn't show the items
that it's supposed to reference"*, and on how: *"discord 'posts' land as normal
and only hidden from the post view they're posted the same day. the nested
items on the unified post are a duplicate or reference of existing content."*
Two properties carry the whole design, and most of what follows pins them:
* NOTHING MOVES. A referenced Discord post keeps its row, its date and its
place in the feed; only a drop sitting beside its own teaser is left out.
* A family is the creator's LEADING working name, one hop from the seed, inside
a window. The refusals are the measured false positives, not invented ones.
"""
from collections import Counter
from datetime import UTC, datetime, timedelta
import pytest
from sqlalchemy import select
from backend.app.models import (
Artist,
ImageRecord,
ImportSettings,
Post,
PostAssociation,
Source,
)
from backend.app.services.discord_grouping import DROP_GROUPER
from backend.app.services.post_association_service import PostAssociationService
from backend.app.services.post_feed_service import PostFeedService
from backend.app.services.post_unification import (
FAMILY_MAX_POSTS,
Candidate,
family,
)
T0 = datetime(2026, 9, 1, 12, 0, tzinfo=UTC)
WINDOW = timedelta(days=60)
def _c(image_id, name, *, days=0, post_id=None, phash=None):
"""One image, Discord-shaped, `days` from T0."""
return Candidate(
image_id=image_id,
post_id=post_id if post_id is not None else image_id,
path=f"20260901_12345678901234{image_id:04d}_01_{name}.png",
phash=phash,
at=T0 + timedelta(days=days),
)
def _family(seed, pool, *, names=None, hashes=None):
return family(
seed, pool, Counter(names or {}), Counter(hashes or {}),
anchor=T0, window=WINDOW,
)
# --- the family rule, pure ---------------------------------------------------
def test_the_older_wips_of_the_same_piece_are_its_family():
"""The operator's ask exactly: *"yellowroom trickles out variants and I want
them to show in the grouped post even if they're older"*. Measured shape:
`Year_20k_wip1 -> wip3 -> Base -> Cndm` over 44 days."""
seed = [_c(1, "Year_20k_Cndm")]
pool = [_c(2, "Year_20k_wip1", days=-44), _c(3, "Year_20K_wip3", days=-44),
_c(4, "Year_20k_Base", days=-20)]
assert [c.image_id for c in _family(seed, pool)] == [2, 3, 4]
def test_a_family_reads_oldest_first():
"""So the card shows the trickle in the order it happened."""
seed = [_c(1, "svtt_drench_b")]
pool = [_c(2, "svtt_wip5", days=-1), _c(3, "svtt_wip1", days=-3)]
assert [c.image_id for c in _family(seed, pool)] == [3, 2]
def test_a_namesake_outside_the_window_is_not_family():
"""Time does most of the work. Measured on artist 8: every collision found
spreads over 500 days — `ashley` 1258, `anya` 1217, `image0` 1974."""
seed = [_c(1, "Ashley_TAIGA")]
pool = [_c(2, "Ashley_Re4_A", days=-1258)]
assert _family(seed, pool) == []
def test_a_shared_trailing_word_is_not_family():
"""The one plain collision inside 60 days on artist 8: `bottom`, three
posts, 56 days apart. No frequency cap can refuse a word that rare — the
leading-name rule does."""
seed = [_c(1, "Undyne_insert_bottom_only-C")]
pool = [_c(2, "Lichgalclc_Lingerie_Bottom_21", days=-56)]
assert _family(seed, pool) == []
def test_a_leading_name_the_creator_uses_everywhere_is_not_a_family():
"""A character is a habit, not a piece. At the cap the name is gated even
inside the window."""
seed = [_c(1, "Bea_Machamp_Shiny")]
pool = [_c(2, "Bea_Machoke_Shiny", days=-5)]
assert _family(seed, pool, names={"bea": FAMILY_MAX_POSTS}) == []
def test_the_family_cap_admits_the_measured_long_families():
"""`tentacooler` spans 6 posts over 7 days and `0-k1` 6 over 10 — real
families, both lost at the pairing cap of 6. That is why the family cap is
its own number."""
seed = [_c(1, "Tentacooler")]
pool = [_c(2, "Tentacooler_c_ins", days=-7)]
assert [c.image_id for c in _family(seed, pool, names={"tentacooler": 6})] == [2]
def test_a_near_duplicate_joins_the_family_without_a_name():
"""Half of this creator's teasers are screenshots, which carry no name. The
same file re-posted is still the same file."""
seed = [_c(1, "image0", phash=0b1011)]
pool = [_c(2, "image0", days=-10, phash=0b1010)]
assert [c.image_id for c in _family(seed, pool)] == [2]
def test_a_distant_hash_is_not_a_duplicate():
"""Lesson #4400: same-artist images are similar whether or not they are the
same piece, so only a NEAR-duplicate counts — the matcher's own line."""
seed = [_c(1, "image0", phash=0)]
pool = [_c(2, "image0", days=-10, phash=(1 << 100) - 1)]
assert _family(seed, pool) == []
def test_a_family_is_one_hop_from_the_seed():
"""Chaining is what lets a family drift: `alpha` reaches an image that also
carries `beta`, and `beta` must not then reach its own family."""
seed = [_c(1, "alpha_final")]
pool = [_c(2, "alpha_beta", days=-3), _c(3, "beta_wip1", days=-4)]
assert [c.image_id for c in _family(seed, pool)] == [2]
def test_the_seed_never_comes_back_as_its_own_family():
seed = [_c(1, "Year_20k_Cndm")]
assert _family(seed, seed + [_c(2, "Year_20k_Base", days=-20)])[0].image_id == 2
assert len(_family(seed, seed)) == 0
# --- the card, end to end ----------------------------------------------------
async def _channels(db, name):
artist = Artist(name=name, slug=name)
db.add(artist)
await db.flush()
patreon = Source(artist_id=artist.id, platform="patreon",
url=f"https://patreon.com/{name}", enabled=True)
discord = Source(artist_id=artist.id, platform="discord",
url=f"https://discord.com/channels/1/{name}", enabled=True)
db.add_all([patreon, discord])
await db.flush()
return artist, patreon, discord
_seq = iter(range(1, 10_000))
async def _post(db, artist, source, *, at, names, body=None, title=None, synthetic=False):
post = Post(
source_id=source.id, artist_id=artist.id,
external_post_id=f"ext-{next(_seq)}", post_date=at,
post_title=title, description=body,
synthesized_by=DROP_GROUPER if synthetic else None,
)
db.add(post)
await db.flush()
for name in names:
n = next(_seq)
db.add(ImageRecord(
# Discord-shaped whatever the platform: the prefix is stripped, so
# the NAME below is the leading name.
path=f"/images/{artist.slug}/20260901_1234567890{n:06d}_01_{name}.png",
sha256=f"{n:064d}", size_bytes=10, mime="image/png",
width=10, height=10, origin="downloaded",
primary_post_id=post.id, artist_id=artist.id,
))
await db.flush()
return post
async def _link(db, teaser, drop, *, status="linked", linked_by="fc"):
a = PostAssociation(
announcement_post_id=teaser.id, payload_post_id=drop.id, score=1.0,
signals={"identity": 1.0, "identity_token": "year"},
status=status, linked_by=linked_by if status == "linked" else None,
)
db.add(a)
await db.flush()
return a
async def _feed(db, artist):
page = await PostFeedService(db).scroll(artist_id=artist.id, limit=100)
return {item["id"]: item for item in page["items"]}
@pytest.mark.integration
@pytest.mark.asyncio
async def test_a_teaser_shows_the_drop_it_announced(db):
"""The complaint itself: the card now carries the drop's images and text."""
artist, patreon, discord = await _channels(db, "unifyartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"],
title="Year 20k", body="Full set in the server")
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1),
names=["Year_20k_Cndm", "Year_20k_Cndm_alt"],
body="@everyone 🍈🍈 the full set", synthetic=True)
assoc = await _link(db, teaser, drop)
await db.commit()
unified = (await _feed(db, artist))[teaser.id]["unified"]
assert [t["role"] for t in unified["thumbnails"]] == ["drop", "drop"]
assert {t["post_id"] for t in unified["thumbnails"]} == {drop.id}
assert unified["links"] == [{
"association_id": assoc.id, "post_id": drop.id,
"linked_by": "fc", "token": "year",
}]
assert unified["texts"][0]["text"] == "@everyone 🍈🍈 the full set"
@pytest.mark.integration
@pytest.mark.asyncio
async def test_a_drop_beside_its_teaser_leaves_the_feed(db):
"""*"only hidden from the post view they're posted the same day"*."""
artist, patreon, discord = await _channels(db, "foldartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=2),
names=["Year_20k_Cndm"], synthetic=True)
await _link(db, teaser, drop)
await db.commit()
feed = await _feed(db, artist)
assert teaser.id in feed
assert drop.id not in feed
# Left out of the FEED, not out of existence: reachable by id, as every
# post is, because it is still the images' true origin.
assert (await PostFeedService(db).get_post(drop.id))["id"] == drop.id
@pytest.mark.integration
@pytest.mark.asyncio
async def test_a_drop_days_from_its_teaser_keeps_its_place(db):
"""A reference does not take anything out of history."""
artist, patreon, discord = await _channels(db, "keepartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(days=3),
names=["Year_20k_Cndm"], synthetic=True)
await _link(db, teaser, drop)
await db.commit()
feed = await _feed(db, artist)
assert drop.id in feed
assert feed[teaser.id]["unified"]["thumbnails"][0]["post_id"] == drop.id
@pytest.mark.integration
@pytest.mark.asyncio
async def test_a_proposal_changes_nothing_on_the_card(db):
"""Only a LINKED pair unifies. A pending proposal is a question for the
review queue, and rendering it would assert a link nobody made."""
artist, patreon, discord = await _channels(db, "pendingartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1),
names=["Year_20k_Cndm"], synthetic=True)
await _link(db, teaser, drop, status="pending")
await db.commit()
feed = await _feed(db, artist)
assert feed[teaser.id]["unified"] is None
assert drop.id in feed
@pytest.mark.integration
@pytest.mark.asyncio
async def test_the_older_variants_come_along_and_nothing_else(db):
"""The family reaches back 44 days for the wips — and not 90 days for a
namesake, and not at all for a message that only shares a trailing word."""
artist, patreon, discord = await _channels(db, "familyartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1),
names=["Year_20k_Cndm"], synthetic=True)
wip = await _post(db, artist, discord, at=T0 - timedelta(days=44),
names=["Year_20k_wip1"], body="wip, feedback welcome")
base = await _post(db, artist, discord, at=T0 - timedelta(days=20),
names=["Year_20K_Base"])
too_old = await _post(db, artist, discord, at=T0 - timedelta(days=90),
names=["Year_20k_old"])
trailing = await _post(db, artist, discord, at=T0 - timedelta(days=5),
names=["Other_piece_year"])
await _link(db, teaser, drop)
await db.commit()
feed = await _feed(db, artist)
unified = feed[teaser.id]["unified"]
variants = [t["post_id"] for t in unified["thumbnails"] if t["role"] == "variant"]
assert variants == [wip.id, base.id]
assert unified["variant_count"] == 2
assert too_old.id not in variants and trailing.id not in variants
assert "wip, feedback welcome" in [t["text"] for t in unified["texts"]]
# Referenced, not moved: every variant is still in the feed on its own.
assert {wip.id, base.id} <= set(feed)
@pytest.mark.integration
@pytest.mark.asyncio
async def test_the_family_window_is_the_operators_setting(db):
artist, patreon, discord = await _channels(db, "windowartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1),
names=["Year_20k_Cndm"], synthetic=True)
await _post(db, artist, discord, at=T0 - timedelta(days=20), names=["Year_20k_wip1"])
await _link(db, teaser, drop)
settings = await db.get(ImportSettings, 1)
settings.discord_family_window_days = 10
await db.commit()
unified = (await _feed(db, artist))[teaser.id]["unified"]
assert unified["variant_count"] == 0
@pytest.mark.integration
@pytest.mark.asyncio
async def test_undo_is_a_dismissal_and_everything_returns(db):
"""The operator chose *"nest automatically, with visible undo"*. The undo
is the review queue's own dismiss: the drop comes back to the feed, the
card loses its references, and the dismissed row is what stops the next
sweep linking the pair straight back."""
artist, patreon, discord = await _channels(db, "undoartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1),
names=["Year_20k_Cndm"], synthetic=True)
assoc = await _link(db, teaser, drop)
await db.commit()
await PostAssociationService(db).dismiss(assoc.id)
await db.commit()
feed = await _feed(db, artist)
assert feed[teaser.id]["unified"] is None
assert drop.id in feed
row = (await db.execute(select(PostAssociation))).scalar_one()
assert (row.status, row.linked_by) == ("dismissed", None)
@pytest.mark.integration
@pytest.mark.asyncio
async def test_an_operator_accept_is_recorded_as_theirs(db):
"""So the card does not claim FC made a link a person made."""
artist, patreon, discord = await _channels(db, "acceptartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1),
names=["Year_20k_Cndm"], synthetic=True)
assoc = await _link(db, teaser, drop, status="pending")
await db.commit()
await PostAssociationService(db).accept(assoc.id)
await db.commit()
unified = (await _feed(db, artist))[teaser.id]["unified"]
assert unified["links"][0]["linked_by"] == "operator"
@pytest.mark.integration
@pytest.mark.asyncio
async def test_a_fold_window_of_zero_hides_nothing(db):
artist, patreon, discord = await _channels(db, "nofoldartist")
teaser = await _post(db, artist, patreon, at=T0, names=["Year_20k_teaser"])
drop = await _post(db, artist, discord, at=T0 - timedelta(hours=1),
names=["Year_20k_Cndm"], synthetic=True)
await _link(db, teaser, drop)
settings = await db.get(ImportSettings, 1)
settings.discord_link_fold_hours = 0
await db.commit()
assert drop.id in await _feed(db, artist)