Release: dev → main (first public release) #258
@@ -59,14 +59,23 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import math
|
import math
|
||||||
|
from collections import Counter
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
from sqlalchemy import Select, func, select, update
|
from sqlalchemy import Select, delete, func, select, update
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ..models import ImageProvenance, ImageRecord, MLSettings, Post, Source
|
from ..models import (
|
||||||
|
ImageProvenance,
|
||||||
|
ImageRecord,
|
||||||
|
MLSettings,
|
||||||
|
Post,
|
||||||
|
PostAssociation,
|
||||||
|
Source,
|
||||||
|
)
|
||||||
|
from .post_naming import FAMILY_MAX_POSTS, leading_name, rarity, token_frequencies
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -600,6 +609,344 @@ async def join_open_groups(
|
|||||||
return joined
|
return joined
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# #4390: a trickle is one drop — later drops of the same piece merge in.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# Operator, 2026-09-24, on a feed of "Grouped from 1 Discord message" cards:
|
||||||
|
# *"the groups are still single image even when they can clearly be seen as
|
||||||
|
# group"*. 665 of Yellowroom's 714 drops were one message.
|
||||||
|
#
|
||||||
|
# Both paths above join on cosine distance to the group's SEED, and the stages
|
||||||
|
# of one piece fail it: `svtt_wip4` did not join `svtt_wip3` from the day
|
||||||
|
# before. A creator trickles a piece out as sketch -> wip -> wip -> release, and
|
||||||
|
# each stage is nearest to the one before it, not to the first.
|
||||||
|
#
|
||||||
|
# Measured on artist 8 before any of this was written (#4390 log):
|
||||||
|
#
|
||||||
|
# * phash cannot see it. Stages of one piece sit 68-134 bits apart; unrelated
|
||||||
|
# pieces by the same artist sit at a median of 126, p5 110. Lesson #4400.
|
||||||
|
# * The embedding's NEAREST neighbour can. Every stage of three real trickles
|
||||||
|
# had a sibling stage as its single nearest image in the artist's whole
|
||||||
|
# library, while siblings further along ranked 40-100 — which is exactly why
|
||||||
|
# seed distance fails. Negative control over all 137 recent Discord images:
|
||||||
|
# where the nearest neighbour was another message within 7 days, the two
|
||||||
|
# carried the same working name 53 times out of 53. Disagreements start past
|
||||||
|
# 7 days.
|
||||||
|
# * The working name sees it directly, when there is one.
|
||||||
|
#
|
||||||
|
# So a later drop merges into an earlier one when the two are within
|
||||||
|
# `discord_group_close_after_hours` of each other (168h — the measured 7 days)
|
||||||
|
# AND either they share a gated LEADING working name, or one's image has the
|
||||||
|
# other's image as its nearest neighbour. A drop reaching SEVERAL earlier drops
|
||||||
|
# pulls them all together — unless two of them are named as different pieces,
|
||||||
|
# in which case nothing moves (see `_compatible`): leaving a drop alone is
|
||||||
|
# recoverable, a wrong merge asserts that unrelated art belongs together.
|
||||||
|
#
|
||||||
|
# Chaining is permitted here and was forbidden above, deliberately. The seed
|
||||||
|
# rule exists because tiny steps can drift from one piece to another; the
|
||||||
|
# measured precision of nearest-neighbour inside 7 days is what bounds drift
|
||||||
|
# for this route, and each link is between neighbours in time, never across a
|
||||||
|
# quiet week.
|
||||||
|
|
||||||
|
# How many unchecked drops one sweep examines per source. A first run over an
|
||||||
|
# established library drains over successive sweeps, oldest first, rather than
|
||||||
|
# issuing one nearest-neighbour query per image of the whole history at once.
|
||||||
|
TRICKLE_BATCH = 300
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _Drop:
|
||||||
|
post: Post
|
||||||
|
members: set[int]
|
||||||
|
first_at: datetime
|
||||||
|
last_at: datetime
|
||||||
|
names: set[str]
|
||||||
|
images: list[int]
|
||||||
|
nearest: set[int] | None
|
||||||
|
|
||||||
|
|
||||||
|
async def _nearest_message(
|
||||||
|
session: AsyncSession, *, artist_id: int, image_id: int, exclude: set[int],
|
||||||
|
) -> int | None:
|
||||||
|
"""The post that owns the nearest image in the artist's whole library.
|
||||||
|
|
||||||
|
The whole LIBRARY, not this source, because that is what was measured: a
|
||||||
|
neighbour that turns out to be a Patreon re-post simply yields no Discord
|
||||||
|
drop to merge into, which errs toward leaving things alone. `exclude` is
|
||||||
|
the drop's own messages — an image is always nearest to its own siblings
|
||||||
|
in the same drop, which says nothing.
|
||||||
|
"""
|
||||||
|
embedding = (await session.execute(
|
||||||
|
select(ImageRecord.siglip_embedding).where(ImageRecord.id == image_id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if embedding is None:
|
||||||
|
return None
|
||||||
|
stmt = (
|
||||||
|
select(ImageRecord.primary_post_id)
|
||||||
|
.where(
|
||||||
|
ImageRecord.artist_id == artist_id,
|
||||||
|
ImageRecord.id != image_id,
|
||||||
|
ImageRecord.siglip_embedding.is_not(None),
|
||||||
|
ImageRecord.primary_post_id.is_not(None),
|
||||||
|
)
|
||||||
|
.order_by(ImageRecord.siglip_embedding.cosine_distance(embedding))
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
if exclude:
|
||||||
|
stmt = stmt.where(ImageRecord.primary_post_id.not_in(exclude))
|
||||||
|
return (await session.execute(stmt)).scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
async def _load_drops(session: AsyncSession, source: Source) -> list[_Drop]:
|
||||||
|
"""Every live drop of this source, with what the merge rule reads, oldest first."""
|
||||||
|
posts = (await session.execute(
|
||||||
|
select(Post).where(
|
||||||
|
Post.source_id == source.id,
|
||||||
|
Post.synthesized_by == DROP_GROUPER,
|
||||||
|
Post.absorbed_by_post_id.is_(None),
|
||||||
|
)
|
||||||
|
)).scalars().all()
|
||||||
|
if not posts:
|
||||||
|
return []
|
||||||
|
by_id = {p.id: p for p in posts}
|
||||||
|
|
||||||
|
msg_at = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||||
|
members: dict[int, set[int]] = {pid: set() for pid in by_id}
|
||||||
|
times: dict[int, list[datetime]] = {pid: [] for pid in by_id}
|
||||||
|
for mid, owner, at in (await session.execute(
|
||||||
|
select(Post.id, Post.absorbed_by_post_id, msg_at)
|
||||||
|
.where(Post.absorbed_by_post_id.in_(list(by_id)))
|
||||||
|
)).all():
|
||||||
|
members[owner].add(mid)
|
||||||
|
times[owner].append(at)
|
||||||
|
|
||||||
|
owner_of = {m: d for d, ms in members.items() for m in ms}
|
||||||
|
names: dict[int, set[str]] = {pid: set() for pid in by_id}
|
||||||
|
images: dict[int, list[int]] = {pid: [] for pid in by_id}
|
||||||
|
if owner_of:
|
||||||
|
for iid, primary, path in (await session.execute(
|
||||||
|
select(ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path)
|
||||||
|
.where(ImageRecord.primary_post_id.in_(list(owner_of)))
|
||||||
|
.order_by(ImageRecord.id)
|
||||||
|
)).all():
|
||||||
|
drop = owner_of[primary]
|
||||||
|
images[drop].append(iid)
|
||||||
|
if (name := leading_name(path)) is not None:
|
||||||
|
names[drop].add(name)
|
||||||
|
|
||||||
|
out = []
|
||||||
|
for pid, post in by_id.items():
|
||||||
|
if not times[pid]:
|
||||||
|
continue
|
||||||
|
stored = (post.synthesis_details or {}).get("nearest_message_ids")
|
||||||
|
out.append(_Drop(
|
||||||
|
post=post, members=members[pid],
|
||||||
|
first_at=min(times[pid]), last_at=max(times[pid]),
|
||||||
|
names=names[pid], images=images[pid],
|
||||||
|
nearest=set(stored) if stored is not None else None,
|
||||||
|
))
|
||||||
|
return sorted(out, key=lambda d: (d.first_at, d.post.id))
|
||||||
|
|
||||||
|
|
||||||
|
async def _name_posts(session: AsyncSession, artist_id: int) -> Counter[str]:
|
||||||
|
"""Post-span counts of the artist's working names — the same corpus the
|
||||||
|
teaser card and the announcement matcher count against."""
|
||||||
|
by_post: dict[int, list[str]] = {}
|
||||||
|
for pid, path in (await session.execute(
|
||||||
|
select(ImageRecord.primary_post_id, ImageRecord.path).where(
|
||||||
|
ImageRecord.artist_id == artist_id,
|
||||||
|
ImageRecord.primary_post_id.is_not(None),
|
||||||
|
)
|
||||||
|
)).all():
|
||||||
|
by_post.setdefault(pid, []).append(path)
|
||||||
|
return token_frequencies(by_post.values())
|
||||||
|
|
||||||
|
|
||||||
|
async def _repoint_associations(
|
||||||
|
session: AsyncSession, *, from_id: int, to_id: int,
|
||||||
|
) -> None:
|
||||||
|
"""Move announcement links from a drop about to merge onto the one it joins.
|
||||||
|
|
||||||
|
Without this the merge would silently undo a teaser link: the association's
|
||||||
|
payload FK cascades on delete. Where the teaser already points at the
|
||||||
|
surviving drop, the stronger claim is kept — a link over a proposal over a
|
||||||
|
dismissal — and the duplicate goes.
|
||||||
|
"""
|
||||||
|
rank = {"linked": 2, "pending": 1, "dismissed": 0}
|
||||||
|
moving = (await session.execute(
|
||||||
|
select(PostAssociation).where(PostAssociation.payload_post_id == from_id)
|
||||||
|
)).scalars().all()
|
||||||
|
for a in moving:
|
||||||
|
existing = (await session.execute(
|
||||||
|
select(PostAssociation).where(
|
||||||
|
PostAssociation.announcement_post_id == a.announcement_post_id,
|
||||||
|
PostAssociation.payload_post_id == to_id,
|
||||||
|
)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if existing is None:
|
||||||
|
a.payload_post_id = to_id
|
||||||
|
continue
|
||||||
|
if rank.get(a.status, 0) > rank.get(existing.status, 0):
|
||||||
|
existing.status = a.status
|
||||||
|
existing.linked_by = a.linked_by
|
||||||
|
await session.delete(a)
|
||||||
|
await session.flush()
|
||||||
|
|
||||||
|
|
||||||
|
async def merge_trickles(
|
||||||
|
session: AsyncSession,
|
||||||
|
source: Source,
|
||||||
|
*,
|
||||||
|
gap: timedelta,
|
||||||
|
min_images: int,
|
||||||
|
cooldown: timedelta,
|
||||||
|
batch: int = TRICKLE_BATCH,
|
||||||
|
) -> int:
|
||||||
|
"""Fold later drops of the same piece into the earlier one. Returns merges."""
|
||||||
|
drops = await _load_drops(session, source)
|
||||||
|
if len(drops) < 2:
|
||||||
|
return 0
|
||||||
|
name_posts = await _name_posts(session, source.artist_id)
|
||||||
|
|
||||||
|
def gated(names: set[str]) -> set[str]:
|
||||||
|
return {n for n in names if rarity(name_posts.get(n, 0), FAMILY_MAX_POSTS) > 0}
|
||||||
|
|
||||||
|
alive: list[_Drop] = []
|
||||||
|
merged = 0
|
||||||
|
checked = 0
|
||||||
|
for drop in drops:
|
||||||
|
details = drop.post.synthesis_details or {}
|
||||||
|
if details.get("trickle_checked"):
|
||||||
|
alive.append(drop)
|
||||||
|
continue
|
||||||
|
if checked >= batch:
|
||||||
|
# Unchecked and out of budget: still a candidate for LATER drops'
|
||||||
|
# reverse edges, just not examined itself this run.
|
||||||
|
alive.append(drop)
|
||||||
|
continue
|
||||||
|
checked += 1
|
||||||
|
|
||||||
|
if drop.nearest is None:
|
||||||
|
found: set[int] = set()
|
||||||
|
for iid in drop.images:
|
||||||
|
pid = await _nearest_message(
|
||||||
|
session, artist_id=source.artist_id, image_id=iid,
|
||||||
|
exclude=drop.members,
|
||||||
|
)
|
||||||
|
if pid is not None:
|
||||||
|
found.add(pid)
|
||||||
|
drop.nearest = found
|
||||||
|
|
||||||
|
mine = gated(drop.names)
|
||||||
|
targets: dict[int, tuple[_Drop, str]] = {}
|
||||||
|
for earlier in alive:
|
||||||
|
if drop.first_at - earlier.last_at > gap:
|
||||||
|
continue
|
||||||
|
shared = mine & gated(earlier.names)
|
||||||
|
if shared:
|
||||||
|
targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
|
||||||
|
elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
|
||||||
|
targets[earlier.post.id] = (earlier, "nearest")
|
||||||
|
|
||||||
|
record = dict(details)
|
||||||
|
record["nearest_message_ids"] = sorted(drop.nearest)
|
||||||
|
record["trickle_checked"] = True
|
||||||
|
drop.post.synthesis_details = record
|
||||||
|
|
||||||
|
if not targets or not _compatible(
|
||||||
|
[gated(t.names) for t, _route in targets.values()] + [mine]
|
||||||
|
):
|
||||||
|
alive.append(drop)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Every target is the same piece as this drop, so they are the same
|
||||||
|
# piece as each other: fold them all into the earliest, then this drop.
|
||||||
|
ordered = sorted(targets.values(), key=lambda tr: (tr[0].first_at, tr[0].post.id))
|
||||||
|
into = ordered[0][0]
|
||||||
|
for other, route in ordered[1:]:
|
||||||
|
await _merge_drop(
|
||||||
|
session, into=into, drop=other, route=route,
|
||||||
|
min_images=min_images, cooldown=cooldown,
|
||||||
|
)
|
||||||
|
alive.remove(other)
|
||||||
|
merged += 1
|
||||||
|
await _merge_drop(
|
||||||
|
session, into=into, drop=drop, route=ordered[0][1],
|
||||||
|
min_images=min_images, cooldown=cooldown,
|
||||||
|
)
|
||||||
|
merged += 1
|
||||||
|
return merged
|
||||||
|
|
||||||
|
|
||||||
|
def _compatible(name_sets: list[set[str]]) -> bool:
|
||||||
|
"""May drops carrying these working names become one post?
|
||||||
|
|
||||||
|
Refused only when two of them are NAMED AS DIFFERENT PIECES — both carry a
|
||||||
|
gated name, and they share none. An unnamed drop (a canvas screenshot)
|
||||||
|
fits anywhere, which is the whole of the Marin case: two early stages both
|
||||||
|
nearest to the same later one are one trickle, not an ambiguity.
|
||||||
|
|
||||||
|
What it does NOT refuse is a drop the creator made two pieces in
|
||||||
|
themselves. Measured on artist 8: one November message carries both
|
||||||
|
`AdL01_wip4` and `Year_20k_wip_z4`, so its drop holds both names, and a
|
||||||
|
later drop of either piece joins it on its own name. That is the creator's
|
||||||
|
co-posting carried forward — Discord shows those two together too — not a
|
||||||
|
bridge FC built.
|
||||||
|
"""
|
||||||
|
named = [n for n in name_sets if n]
|
||||||
|
return all(a & b for i, a in enumerate(named) for b in named[i + 1:])
|
||||||
|
|
||||||
|
|
||||||
|
async def _merge_drop(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
into: _Drop,
|
||||||
|
drop: _Drop,
|
||||||
|
route: str,
|
||||||
|
min_images: int,
|
||||||
|
cooldown: timedelta,
|
||||||
|
) -> None:
|
||||||
|
"""Absorb `drop`'s messages into `into`, carry its links over, delete it.
|
||||||
|
|
||||||
|
Growth is stamped at the merged messages' OWN time, not the wall clock.
|
||||||
|
Merging history must not drag a two-year-old drop to the top of the feed,
|
||||||
|
and the time the group actually grew is when those messages arrived.
|
||||||
|
"""
|
||||||
|
await _repoint_associations(session, from_id=drop.post.id, to_id=into.post.id)
|
||||||
|
grew_before = into.post.last_grew_at
|
||||||
|
await _absorb_into(
|
||||||
|
session, group=into.post, member_ids=sorted(drop.members),
|
||||||
|
source_id=into.post.source_id, now=drop.last_at,
|
||||||
|
min_images=min_images, cooldown=cooldown,
|
||||||
|
)
|
||||||
|
# Never backwards: a group that already grew later than these messages
|
||||||
|
# keeps that later date.
|
||||||
|
if grew_before is not None and grew_before > drop.last_at:
|
||||||
|
into.post.last_grew_at = grew_before
|
||||||
|
details = dict(into.post.synthesis_details or {})
|
||||||
|
if grew_before is not None and grew_before > drop.last_at:
|
||||||
|
details["last_grew_at"] = grew_before.isoformat()
|
||||||
|
# The honesty rule, extended: a grouping FC invented says what it was
|
||||||
|
# built from, and a merge says WHY — "name:svtt" or "nearest".
|
||||||
|
details["merged"] = [
|
||||||
|
*details.get("merged", []),
|
||||||
|
{"post_id": drop.post.id, "route": route, "message_ids": sorted(drop.members)},
|
||||||
|
]
|
||||||
|
details["nearest_message_ids"] = sorted((into.nearest or set()) | (drop.nearest or set()))
|
||||||
|
into.post.synthesis_details = details
|
||||||
|
|
||||||
|
into.members |= drop.members
|
||||||
|
into.names |= drop.names
|
||||||
|
into.images += drop.images
|
||||||
|
into.nearest = (into.nearest or set()) | (drop.nearest or set())
|
||||||
|
into.last_at = max(into.last_at, drop.last_at)
|
||||||
|
|
||||||
|
await session.execute(delete(ImageProvenance).where(ImageProvenance.post_id == drop.post.id))
|
||||||
|
await session.delete(drop.post)
|
||||||
|
await session.flush()
|
||||||
|
|
||||||
|
|
||||||
async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||||
"""Group every enabled Discord source. No-op when the switch is off.
|
"""Group every enabled Discord source. No-op when the switch is off.
|
||||||
|
|
||||||
@@ -612,6 +959,7 @@ async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
|||||||
if not settings.discord_grouping_enabled:
|
if not settings.discord_grouping_enabled:
|
||||||
return {
|
return {
|
||||||
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
||||||
|
"drops_merged": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
sources = (await session.execute(
|
sources = (await session.execute(
|
||||||
@@ -626,6 +974,7 @@ async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
|||||||
|
|
||||||
created = 0
|
created = 0
|
||||||
joined = 0
|
joined = 0
|
||||||
|
merged = 0
|
||||||
for source in sources:
|
for source in sources:
|
||||||
joined += await join_open_groups(
|
joined += await join_open_groups(
|
||||||
session, source,
|
session, source,
|
||||||
@@ -644,12 +993,20 @@ async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
|||||||
window_minutes=window_minutes,
|
window_minutes=window_minutes,
|
||||||
now=now,
|
now=now,
|
||||||
)
|
)
|
||||||
|
# Last, so the drops the two passes above just wrote are merged in
|
||||||
|
# the same sweep rather than showing as singletons for an hour.
|
||||||
|
merged += await merge_trickles(
|
||||||
|
session, source,
|
||||||
|
gap=timedelta(hours=float(settings.discord_group_close_after_hours)),
|
||||||
|
min_images=int(settings.discord_group_resurface_min_images),
|
||||||
|
cooldown=timedelta(hours=float(settings.discord_group_resurface_cooldown_hours)),
|
||||||
|
)
|
||||||
log.info(
|
log.info(
|
||||||
"discord drop grouping: %d source(s), %d synthetic post(s) created, "
|
"discord drop grouping: %d source(s), %d synthetic post(s) created, "
|
||||||
"%d image(s) joined to open groups",
|
"%d image(s) joined to open groups, %d trickle drop(s) merged",
|
||||||
len(sources), created, joined,
|
len(sources), created, joined, merged,
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"enabled": True, "sources": len(sources),
|
"enabled": True, "sources": len(sources),
|
||||||
"posts_created": created, "images_joined": joined,
|
"posts_created": created, "images_joined": joined, "drops_merged": merged,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -165,6 +165,19 @@ MAX_TOKEN_POSTS = 6
|
|||||||
# four-post band, where conto's `illustration9` and `maid` also sit.
|
# four-post band, where conto's `illustration9` and `maid` also sit.
|
||||||
IDENTITY_FLOOR = 0.75
|
IDENTITY_FLOOR = 0.75
|
||||||
|
|
||||||
|
# A LEADING name spanning this many of ONE ARTIST's posts is a habit — a
|
||||||
|
# character the creator returns to — not one piece's trickle. Used wherever a
|
||||||
|
# name gathers a FAMILY: the teaser card's variants (#4401) and the Discord
|
||||||
|
# grouper's trickle merge (#4390), so the two cannot disagree about what a
|
||||||
|
# family is.
|
||||||
|
#
|
||||||
|
# Its own value rather than MAX_TOKEN_POSTS (6), which is calibrated for
|
||||||
|
# PAIRING two posts and measured too tight for a family. On artist 8,
|
||||||
|
# `tentacooler` spans 6 posts over 7 days and `0-k1` 6 posts over 10: both real
|
||||||
|
# families, both gated out at 6. At 8, `anya` (7) passes the cap — and has no
|
||||||
|
# pair inside any family window, which is what the window is for.
|
||||||
|
FAMILY_MAX_POSTS = 8
|
||||||
|
|
||||||
|
|
||||||
def _strip_prefixes(stem: str) -> str:
|
def _strip_prefixes(stem: str) -> str:
|
||||||
"""Remove the framing each platform's importer adds around the real name."""
|
"""Remove the framing each platform's importer adds around the real name."""
|
||||||
|
|||||||
@@ -58,17 +58,7 @@ from ..utils.text import html_to_plain, truncate_at_word
|
|||||||
from .discord_grouping import PLATFORM as DISCORD
|
from .discord_grouping import PLATFORM as DISCORD
|
||||||
from .gallery_service import thumbnail_url
|
from .gallery_service import thumbnail_url
|
||||||
from .post_association_service import DUPLICATE_MAX_DISTANCE
|
from .post_association_service import DUPLICATE_MAX_DISTANCE
|
||||||
from .post_naming import leading_name, rarity, token_frequencies
|
from .post_naming import FAMILY_MAX_POSTS, leading_name, rarity, token_frequencies
|
||||||
|
|
||||||
# A leading name spanning this many of ONE ARTIST's posts is a habit — a
|
|
||||||
# character the creator returns to — not one piece's trickle.
|
|
||||||
#
|
|
||||||
# Its own value rather than post_naming.MAX_TOKEN_POSTS (6), which is
|
|
||||||
# calibrated for PAIRING two posts, and measured too tight for a family. On
|
|
||||||
# artist 8, `tentacooler` spans 6 posts over 7 days and `0-k1` 6 posts over 10:
|
|
||||||
# both real families, both gated out at 6. At 8, `anya` (7) passes the cap —
|
|
||||||
# and has no pair inside the family window, which is what the window is for.
|
|
||||||
FAMILY_MAX_POSTS = 8
|
|
||||||
|
|
||||||
# The text each referenced post contributes to the card, per post. The card
|
# The text each referenced post contributes to the card, per post. The card
|
||||||
# clamps it again; this keeps a long Discord thread from making the feed
|
# clamps it again; this keeps a long Discord thread from making the feed
|
||||||
|
|||||||
@@ -1175,7 +1175,7 @@ def group_discord_drops() -> str:
|
|||||||
return "disabled"
|
return "disabled"
|
||||||
return (
|
return (
|
||||||
f"sources={res['sources']} created={res['posts_created']} "
|
f"sources={res['sources']} created={res['posts_created']} "
|
||||||
f"joined={res['images_joined']}"
|
f"joined={res['images_joined']} merged={res['drops_merged']}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -374,6 +374,7 @@ async def test_the_sweep_is_a_no_op_when_the_switch_is_off(db):
|
|||||||
# `images_joined` broke this assertion, which is exactly what it is for.
|
# `images_joined` broke this assertion, which is exactly what it is for.
|
||||||
assert result == {
|
assert result == {
|
||||||
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
||||||
|
"drops_merged": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,253 @@
|
|||||||
|
"""#4390: a creator's trickle is one drop, not a card per message.
|
||||||
|
|
||||||
|
Operator, 2026-09-24, on a feed of "Grouped from 1 Discord message" cards:
|
||||||
|
*"the groups are still single image even when they can clearly be seen as
|
||||||
|
group"*. 665 of Yellowroom's 714 drops were one message, because both join
|
||||||
|
paths demand closeness to a drop's FIRST image and a piece's stages drift away
|
||||||
|
from it — each is nearest the one before, not the first.
|
||||||
|
|
||||||
|
The merge pass joins a later drop to an earlier one within 7 days when they
|
||||||
|
share a working name, or when one's image is the other's nearest neighbour in
|
||||||
|
the artist's library. Vectors here are built at stated angles (see
|
||||||
|
test_discord_grouping._vec) so each test says exactly which image is nearest
|
||||||
|
to which, rather than hoping.
|
||||||
|
"""
|
||||||
|
import math
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import (
|
||||||
|
Artist,
|
||||||
|
ImageRecord,
|
||||||
|
Post,
|
||||||
|
PostAssociation,
|
||||||
|
Source,
|
||||||
|
)
|
||||||
|
from backend.app.services.discord_grouping import (
|
||||||
|
DROP_GROUPER,
|
||||||
|
_compatible,
|
||||||
|
group_source,
|
||||||
|
merge_trickles,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
DIM = 1152
|
||||||
|
GAP = timedelta(hours=168)
|
||||||
|
T0 = datetime.now(UTC) - timedelta(days=30)
|
||||||
|
|
||||||
|
|
||||||
|
def _vec(angle: float) -> list[float]:
|
||||||
|
v = [0.0] * DIM
|
||||||
|
v[0] = math.cos(angle)
|
||||||
|
v[1] = math.sin(angle)
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
# --- the compatibility rule, pure --------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_unnamed_stages_fit_with_anything():
|
||||||
|
"""The Marin case: canvas screenshots carry no name, and two early stages
|
||||||
|
both nearest to one later stage are one trickle, not an ambiguity."""
|
||||||
|
assert _compatible([set(), set(), set()])
|
||||||
|
assert _compatible([set(), {"svtt"}])
|
||||||
|
|
||||||
|
|
||||||
|
def test_two_differently_named_pieces_never_meet():
|
||||||
|
assert not _compatible([{"alpha"}, {"beta"}, set()])
|
||||||
|
|
||||||
|
|
||||||
|
def test_pieces_sharing_a_name_are_one_piece():
|
||||||
|
assert _compatible([{"year", "20k"}, {"year"}])
|
||||||
|
|
||||||
|
|
||||||
|
# --- end to end --------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed(db, name):
|
||||||
|
artist = Artist(name=name, slug=name)
|
||||||
|
db.add(artist)
|
||||||
|
await db.flush()
|
||||||
|
source = Source(artist_id=artist.id, platform="discord",
|
||||||
|
url=f"https://discord.com/channels/1/{name}", enabled=True)
|
||||||
|
db.add(source)
|
||||||
|
await db.flush()
|
||||||
|
return artist, source
|
||||||
|
|
||||||
|
|
||||||
|
_n = iter(range(1, 100_000))
|
||||||
|
|
||||||
|
|
||||||
|
async def _message(db, artist, source, *, at, angle, name=None, text=None):
|
||||||
|
n = next(_n)
|
||||||
|
post = Post(source_id=source.id, artist_id=artist.id,
|
||||||
|
external_post_id=f"msg-{n}", post_date=at, description=text)
|
||||||
|
db.add(post)
|
||||||
|
await db.flush()
|
||||||
|
# gallery-dl's Discord shape, so the prefix strips and NAME is the working
|
||||||
|
# name. Unnamed messages are canvas screenshots, which carry none.
|
||||||
|
stem = name or f"Screenshot_2026-09-08_{n:06d}"
|
||||||
|
db.add(ImageRecord(
|
||||||
|
path=f"/images/{artist.slug}/20260901_{1234567890000 + n}_01_{stem}.png",
|
||||||
|
sha256=f"{n:064d}", size_bytes=10, mime="image/png", width=10, height=10,
|
||||||
|
origin="downloaded", primary_post_id=post.id, artist_id=artist.id,
|
||||||
|
siglip_embedding=_vec(angle),
|
||||||
|
))
|
||||||
|
await db.flush()
|
||||||
|
return post
|
||||||
|
|
||||||
|
|
||||||
|
async def _group_then_merge(db, source):
|
||||||
|
"""E2 makes the singletons exactly as it does live; then the merge pass."""
|
||||||
|
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||||
|
merged = await merge_trickles(
|
||||||
|
db, source, gap=GAP, min_images=2, cooldown=timedelta(hours=24),
|
||||||
|
)
|
||||||
|
await db.commit()
|
||||||
|
return merged
|
||||||
|
|
||||||
|
|
||||||
|
async def _drops(db, source):
|
||||||
|
return (await db.execute(
|
||||||
|
select(Post).where(
|
||||||
|
Post.source_id == source.id,
|
||||||
|
Post.synthesized_by == DROP_GROUPER,
|
||||||
|
Post.absorbed_by_post_id.is_(None),
|
||||||
|
).order_by(Post.post_date)
|
||||||
|
)).scalars().all()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_stages_a_day_apart_become_one_drop(db):
|
||||||
|
"""The screenshot, measured shape: each stage 0.12 from the next and 0.46
|
||||||
|
from the first, so seed distance splits them and nearest-neighbour joins
|
||||||
|
them."""
|
||||||
|
artist, source = await _seed(db, "trickle-artist")
|
||||||
|
a = await _message(db, artist, source, at=T0, angle=0.0, text="Very early Marin.")
|
||||||
|
b = await _message(db, artist, source, at=T0 + timedelta(hours=17), angle=0.5,
|
||||||
|
text="Might get mirrored.")
|
||||||
|
c = await _message(db, artist, source, at=T0 + timedelta(hours=20), angle=1.0,
|
||||||
|
text="Got there eventually.")
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
assert await _group_then_merge(db, source) == 2
|
||||||
|
|
||||||
|
(drop,) = await _drops(db, source)
|
||||||
|
assert set(drop.synthesis_details["member_post_ids"]) == {a.id, b.id, c.id}
|
||||||
|
assert drop.synthesis_details["message_count"] == 3
|
||||||
|
# The body is every message's text, in arrival order.
|
||||||
|
assert drop.description.index("Very early") < drop.description.index("Got there")
|
||||||
|
# And it says why — a grouping FC invented has to be checkable.
|
||||||
|
assert {m["route"] for m in drop.synthesis_details["merged"]} == {"nearest"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_shared_working_name_joins_across_days(db):
|
||||||
|
"""`svtt_wip3` did not join `svtt_wip4` from the day before on the live
|
||||||
|
instance. Orthogonal vectors here, so only the name can do it — and the
|
||||||
|
route it records says so."""
|
||||||
|
artist, source = await _seed(db, "named-artist")
|
||||||
|
await _message(db, artist, source, at=T0, angle=0.0, name="svtt_wip3")
|
||||||
|
await _message(db, artist, source, at=T0 + timedelta(days=2), angle=math.pi / 2,
|
||||||
|
name="svtt_drench_b")
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
await _group_then_merge(db, source)
|
||||||
|
|
||||||
|
(drop,) = await _drops(db, source)
|
||||||
|
assert drop.synthesis_details["merged"][0]["route"] == "name:svtt"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_nothing_joins_across_a_quiet_week(db):
|
||||||
|
"""Measured: where the nearest neighbour was another message within 7 days
|
||||||
|
the names agreed 53 times of 53; past 7 days they begin to disagree."""
|
||||||
|
artist, source = await _seed(db, "quiet-artist")
|
||||||
|
await _message(db, artist, source, at=T0, angle=0.0, name="alpha_wip1")
|
||||||
|
await _message(db, artist, source, at=T0 + timedelta(days=8), angle=0.05,
|
||||||
|
name="alpha_base")
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
assert await _group_then_merge(db, source) == 0
|
||||||
|
assert len(await _drops(db, source)) == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_two_named_pieces_meeting_through_a_third_stay_apart(db):
|
||||||
|
"""C's image is nearest A's; B's image is nearest C's. A is `alpha`, B is
|
||||||
|
`beta` — so C reaches two different pieces and nothing moves."""
|
||||||
|
artist, source = await _seed(db, "bridge-artist")
|
||||||
|
await _message(db, artist, source, at=T0, angle=0.0, name="alpha")
|
||||||
|
await _message(db, artist, source, at=T0 + timedelta(hours=3), angle=0.6, name="beta")
|
||||||
|
await _message(db, artist, source, at=T0 + timedelta(days=1), angle=0.25)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
assert await _group_then_merge(db, source) == 0
|
||||||
|
assert len(await _drops(db, source)) == 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_teaser_link_survives_its_drop_being_merged(db):
|
||||||
|
"""The association's payload FK cascades. Merging the drop a teaser was
|
||||||
|
linked to must carry the link across, or it silently undoes #4402."""
|
||||||
|
artist, source = await _seed(db, "linked-artist")
|
||||||
|
patreon = Source(artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://patreon.com/linked-artist", enabled=True)
|
||||||
|
db.add(patreon)
|
||||||
|
await db.flush()
|
||||||
|
teaser = Post(source_id=patreon.id, artist_id=artist.id, external_post_id="teaser",
|
||||||
|
post_date=T0 + timedelta(days=1, hours=2))
|
||||||
|
db.add(teaser)
|
||||||
|
await _message(db, artist, source, at=T0, angle=0.0, name="svtt_wip3")
|
||||||
|
await _message(db, artist, source, at=T0 + timedelta(days=1), angle=math.pi / 2,
|
||||||
|
name="svtt_drench_b")
|
||||||
|
await db.commit()
|
||||||
|
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||||
|
later = (await _drops(db, source))[-1]
|
||||||
|
db.add(PostAssociation(announcement_post_id=teaser.id, payload_post_id=later.id,
|
||||||
|
score=1.0, status="linked", linked_by="fc"))
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
await merge_trickles(db, source, gap=GAP, min_images=2, cooldown=timedelta(hours=24))
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
(drop,) = await _drops(db, source)
|
||||||
|
link = (await db.execute(select(PostAssociation))).scalar_one()
|
||||||
|
assert (link.payload_post_id, link.status, link.linked_by) == (drop.id, "linked", "fc")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_merging_history_does_not_drag_it_to_the_top_of_the_feed(db):
|
||||||
|
"""Growth is stamped at the merged messages' own time. A merge the first
|
||||||
|
sweep makes over two-year-old drops must not read as news today."""
|
||||||
|
artist, source = await _seed(db, "history-artist")
|
||||||
|
old = datetime.now(UTC) - timedelta(days=700)
|
||||||
|
await _message(db, artist, source, at=old, angle=0.0, name="svtt_wip1")
|
||||||
|
await _message(db, artist, source, at=old + timedelta(days=1), angle=0.5, name="svtt_wip2")
|
||||||
|
await _message(db, artist, source, at=old + timedelta(days=2), angle=1.0, name="svtt_base")
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
await _group_then_merge(db, source)
|
||||||
|
|
||||||
|
(drop,) = await _drops(db, source)
|
||||||
|
assert drop.last_grew_at <= old + timedelta(days=2, minutes=1)
|
||||||
|
assert drop.resurfaced_at is None or drop.resurfaced_at <= old + timedelta(days=2, minutes=1)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_checked_drop_is_not_examined_again(db):
|
||||||
|
"""Each drop costs one nearest-neighbour query per image, once. The flag is
|
||||||
|
what stops every hourly sweep repeating the whole history."""
|
||||||
|
artist, source = await _seed(db, "checked-artist")
|
||||||
|
await _message(db, artist, source, at=T0, angle=0.0, name="alpha")
|
||||||
|
await _message(db, artist, source, at=T0 + timedelta(days=3), angle=math.pi / 2,
|
||||||
|
name="beta")
|
||||||
|
await db.commit()
|
||||||
|
await _group_then_merge(db, source)
|
||||||
|
|
||||||
|
for drop in await _drops(db, source):
|
||||||
|
assert drop.synthesis_details["trickle_checked"] is True
|
||||||
|
assert "nearest_message_ids" in drop.synthesis_details
|
||||||
@@ -29,11 +29,8 @@ from backend.app.models import (
|
|||||||
from backend.app.services.discord_grouping import DROP_GROUPER
|
from backend.app.services.discord_grouping import DROP_GROUPER
|
||||||
from backend.app.services.post_association_service import PostAssociationService
|
from backend.app.services.post_association_service import PostAssociationService
|
||||||
from backend.app.services.post_feed_service import PostFeedService
|
from backend.app.services.post_feed_service import PostFeedService
|
||||||
from backend.app.services.post_unification import (
|
from backend.app.services.post_naming import FAMILY_MAX_POSTS
|
||||||
FAMILY_MAX_POSTS,
|
from backend.app.services.post_unification import Candidate, family
|
||||||
Candidate,
|
|
||||||
family,
|
|
||||||
)
|
|
||||||
|
|
||||||
T0 = datetime(2026, 9, 1, 12, 0, tzinfo=UTC)
|
T0 = datetime(2026, 9, 1, 12, 0, tzinfo=UTC)
|
||||||
WINDOW = timedelta(days=60)
|
WINDOW = timedelta(days=60)
|
||||||
|
|||||||
Reference in New Issue
Block a user