Release: dev → main (first public release) #258
@@ -59,14 +59,23 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import Select, func, select, update
|
||||
from sqlalchemy import Select, delete, func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import ImageProvenance, ImageRecord, MLSettings, Post, Source
|
||||
from ..models import (
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
MLSettings,
|
||||
Post,
|
||||
PostAssociation,
|
||||
Source,
|
||||
)
|
||||
from .post_naming import FAMILY_MAX_POSTS, leading_name, rarity, token_frequencies
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -600,6 +609,344 @@ async def join_open_groups(
|
||||
return joined
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #4390: a trickle is one drop — later drops of the same piece merge in.
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Operator, 2026-09-24, on a feed of "Grouped from 1 Discord message" cards:
|
||||
# *"the groups are still single image even when they can clearly be seen as
|
||||
# group"*. 665 of Yellowroom's 714 drops were one message.
|
||||
#
|
||||
# Both paths above join on cosine distance to the group's SEED, and the stages
|
||||
# of one piece fail it: `svtt_wip4` did not join `svtt_wip3` from the day
|
||||
# before. A creator trickles a piece out as sketch -> wip -> wip -> release, and
|
||||
# each stage is nearest to the one before it, not to the first.
|
||||
#
|
||||
# Measured on artist 8 before any of this was written (#4390 log):
|
||||
#
|
||||
# * phash cannot see it. Stages of one piece sit 68-134 bits apart; unrelated
|
||||
# pieces by the same artist sit at a median of 126, p5 110. Lesson #4400.
|
||||
# * The embedding's NEAREST neighbour can. Every stage of three real trickles
|
||||
# had a sibling stage as its single nearest image in the artist's whole
|
||||
# library, while siblings further along ranked 40-100 — which is exactly why
|
||||
# seed distance fails. Negative control over all 137 recent Discord images:
|
||||
# where the nearest neighbour was another message within 7 days, the two
|
||||
# carried the same working name 53 times out of 53. Disagreements start past
|
||||
# 7 days.
|
||||
# * The working name sees it directly, when there is one.
|
||||
#
|
||||
# So a later drop merges into an earlier one when the two are within
|
||||
# `discord_group_close_after_hours` of each other (168h — the measured 7 days)
|
||||
# AND either they share a gated LEADING working name, or one's image has the
|
||||
# other's image as its nearest neighbour. A drop reaching SEVERAL earlier drops
|
||||
# pulls them all together — unless two of them are named as different pieces,
|
||||
# in which case nothing moves (see `_compatible`): leaving a drop alone is
|
||||
# recoverable, a wrong merge asserts that unrelated art belongs together.
|
||||
#
|
||||
# Chaining is permitted here and was forbidden above, deliberately. The seed
|
||||
# rule exists because tiny steps can drift from one piece to another; the
|
||||
# measured precision of nearest-neighbour inside 7 days is what bounds drift
|
||||
# for this route, and each link is between neighbours in time, never across a
|
||||
# quiet week.
|
||||
|
||||
# How many unchecked drops one sweep examines per source. A first run over an
|
||||
# established library drains over successive sweeps, oldest first, rather than
|
||||
# issuing one nearest-neighbour query per image of the whole history at once.
|
||||
TRICKLE_BATCH = 300
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Drop:
|
||||
post: Post
|
||||
members: set[int]
|
||||
first_at: datetime
|
||||
last_at: datetime
|
||||
names: set[str]
|
||||
images: list[int]
|
||||
nearest: set[int] | None
|
||||
|
||||
|
||||
async def _nearest_message(
|
||||
session: AsyncSession, *, artist_id: int, image_id: int, exclude: set[int],
|
||||
) -> int | None:
|
||||
"""The post that owns the nearest image in the artist's whole library.
|
||||
|
||||
The whole LIBRARY, not this source, because that is what was measured: a
|
||||
neighbour that turns out to be a Patreon re-post simply yields no Discord
|
||||
drop to merge into, which errs toward leaving things alone. `exclude` is
|
||||
the drop's own messages — an image is always nearest to its own siblings
|
||||
in the same drop, which says nothing.
|
||||
"""
|
||||
embedding = (await session.execute(
|
||||
select(ImageRecord.siglip_embedding).where(ImageRecord.id == image_id)
|
||||
)).scalar_one_or_none()
|
||||
if embedding is None:
|
||||
return None
|
||||
stmt = (
|
||||
select(ImageRecord.primary_post_id)
|
||||
.where(
|
||||
ImageRecord.artist_id == artist_id,
|
||||
ImageRecord.id != image_id,
|
||||
ImageRecord.siglip_embedding.is_not(None),
|
||||
ImageRecord.primary_post_id.is_not(None),
|
||||
)
|
||||
.order_by(ImageRecord.siglip_embedding.cosine_distance(embedding))
|
||||
.limit(1)
|
||||
)
|
||||
if exclude:
|
||||
stmt = stmt.where(ImageRecord.primary_post_id.not_in(exclude))
|
||||
return (await session.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
|
||||
async def _load_drops(session: AsyncSession, source: Source) -> list[_Drop]:
|
||||
"""Every live drop of this source, with what the merge rule reads, oldest first."""
|
||||
posts = (await session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == source.id,
|
||||
Post.synthesized_by == DROP_GROUPER,
|
||||
Post.absorbed_by_post_id.is_(None),
|
||||
)
|
||||
)).scalars().all()
|
||||
if not posts:
|
||||
return []
|
||||
by_id = {p.id: p for p in posts}
|
||||
|
||||
msg_at = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||
members: dict[int, set[int]] = {pid: set() for pid in by_id}
|
||||
times: dict[int, list[datetime]] = {pid: [] for pid in by_id}
|
||||
for mid, owner, at in (await session.execute(
|
||||
select(Post.id, Post.absorbed_by_post_id, msg_at)
|
||||
.where(Post.absorbed_by_post_id.in_(list(by_id)))
|
||||
)).all():
|
||||
members[owner].add(mid)
|
||||
times[owner].append(at)
|
||||
|
||||
owner_of = {m: d for d, ms in members.items() for m in ms}
|
||||
names: dict[int, set[str]] = {pid: set() for pid in by_id}
|
||||
images: dict[int, list[int]] = {pid: [] for pid in by_id}
|
||||
if owner_of:
|
||||
for iid, primary, path in (await session.execute(
|
||||
select(ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path)
|
||||
.where(ImageRecord.primary_post_id.in_(list(owner_of)))
|
||||
.order_by(ImageRecord.id)
|
||||
)).all():
|
||||
drop = owner_of[primary]
|
||||
images[drop].append(iid)
|
||||
if (name := leading_name(path)) is not None:
|
||||
names[drop].add(name)
|
||||
|
||||
out = []
|
||||
for pid, post in by_id.items():
|
||||
if not times[pid]:
|
||||
continue
|
||||
stored = (post.synthesis_details or {}).get("nearest_message_ids")
|
||||
out.append(_Drop(
|
||||
post=post, members=members[pid],
|
||||
first_at=min(times[pid]), last_at=max(times[pid]),
|
||||
names=names[pid], images=images[pid],
|
||||
nearest=set(stored) if stored is not None else None,
|
||||
))
|
||||
return sorted(out, key=lambda d: (d.first_at, d.post.id))
|
||||
|
||||
|
||||
async def _name_posts(session: AsyncSession, artist_id: int) -> Counter[str]:
|
||||
"""Post-span counts of the artist's working names — the same corpus the
|
||||
teaser card and the announcement matcher count against."""
|
||||
by_post: dict[int, list[str]] = {}
|
||||
for pid, path in (await session.execute(
|
||||
select(ImageRecord.primary_post_id, ImageRecord.path).where(
|
||||
ImageRecord.artist_id == artist_id,
|
||||
ImageRecord.primary_post_id.is_not(None),
|
||||
)
|
||||
)).all():
|
||||
by_post.setdefault(pid, []).append(path)
|
||||
return token_frequencies(by_post.values())
|
||||
|
||||
|
||||
async def _repoint_associations(
|
||||
session: AsyncSession, *, from_id: int, to_id: int,
|
||||
) -> None:
|
||||
"""Move announcement links from a drop about to merge onto the one it joins.
|
||||
|
||||
Without this the merge would silently undo a teaser link: the association's
|
||||
payload FK cascades on delete. Where the teaser already points at the
|
||||
surviving drop, the stronger claim is kept — a link over a proposal over a
|
||||
dismissal — and the duplicate goes.
|
||||
"""
|
||||
rank = {"linked": 2, "pending": 1, "dismissed": 0}
|
||||
moving = (await session.execute(
|
||||
select(PostAssociation).where(PostAssociation.payload_post_id == from_id)
|
||||
)).scalars().all()
|
||||
for a in moving:
|
||||
existing = (await session.execute(
|
||||
select(PostAssociation).where(
|
||||
PostAssociation.announcement_post_id == a.announcement_post_id,
|
||||
PostAssociation.payload_post_id == to_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if existing is None:
|
||||
a.payload_post_id = to_id
|
||||
continue
|
||||
if rank.get(a.status, 0) > rank.get(existing.status, 0):
|
||||
existing.status = a.status
|
||||
existing.linked_by = a.linked_by
|
||||
await session.delete(a)
|
||||
await session.flush()
|
||||
|
||||
|
||||
async def merge_trickles(
|
||||
session: AsyncSession,
|
||||
source: Source,
|
||||
*,
|
||||
gap: timedelta,
|
||||
min_images: int,
|
||||
cooldown: timedelta,
|
||||
batch: int = TRICKLE_BATCH,
|
||||
) -> int:
|
||||
"""Fold later drops of the same piece into the earlier one. Returns merges."""
|
||||
drops = await _load_drops(session, source)
|
||||
if len(drops) < 2:
|
||||
return 0
|
||||
name_posts = await _name_posts(session, source.artist_id)
|
||||
|
||||
def gated(names: set[str]) -> set[str]:
|
||||
return {n for n in names if rarity(name_posts.get(n, 0), FAMILY_MAX_POSTS) > 0}
|
||||
|
||||
alive: list[_Drop] = []
|
||||
merged = 0
|
||||
checked = 0
|
||||
for drop in drops:
|
||||
details = drop.post.synthesis_details or {}
|
||||
if details.get("trickle_checked"):
|
||||
alive.append(drop)
|
||||
continue
|
||||
if checked >= batch:
|
||||
# Unchecked and out of budget: still a candidate for LATER drops'
|
||||
# reverse edges, just not examined itself this run.
|
||||
alive.append(drop)
|
||||
continue
|
||||
checked += 1
|
||||
|
||||
if drop.nearest is None:
|
||||
found: set[int] = set()
|
||||
for iid in drop.images:
|
||||
pid = await _nearest_message(
|
||||
session, artist_id=source.artist_id, image_id=iid,
|
||||
exclude=drop.members,
|
||||
)
|
||||
if pid is not None:
|
||||
found.add(pid)
|
||||
drop.nearest = found
|
||||
|
||||
mine = gated(drop.names)
|
||||
targets: dict[int, tuple[_Drop, str]] = {}
|
||||
for earlier in alive:
|
||||
if drop.first_at - earlier.last_at > gap:
|
||||
continue
|
||||
shared = mine & gated(earlier.names)
|
||||
if shared:
|
||||
targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
|
||||
elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
|
||||
targets[earlier.post.id] = (earlier, "nearest")
|
||||
|
||||
record = dict(details)
|
||||
record["nearest_message_ids"] = sorted(drop.nearest)
|
||||
record["trickle_checked"] = True
|
||||
drop.post.synthesis_details = record
|
||||
|
||||
if not targets or not _compatible(
|
||||
[gated(t.names) for t, _route in targets.values()] + [mine]
|
||||
):
|
||||
alive.append(drop)
|
||||
continue
|
||||
|
||||
# Every target is the same piece as this drop, so they are the same
|
||||
# piece as each other: fold them all into the earliest, then this drop.
|
||||
ordered = sorted(targets.values(), key=lambda tr: (tr[0].first_at, tr[0].post.id))
|
||||
into = ordered[0][0]
|
||||
for other, route in ordered[1:]:
|
||||
await _merge_drop(
|
||||
session, into=into, drop=other, route=route,
|
||||
min_images=min_images, cooldown=cooldown,
|
||||
)
|
||||
alive.remove(other)
|
||||
merged += 1
|
||||
await _merge_drop(
|
||||
session, into=into, drop=drop, route=ordered[0][1],
|
||||
min_images=min_images, cooldown=cooldown,
|
||||
)
|
||||
merged += 1
|
||||
return merged
|
||||
|
||||
|
||||
def _compatible(name_sets: list[set[str]]) -> bool:
|
||||
"""May drops carrying these working names become one post?
|
||||
|
||||
Refused only when two of them are NAMED AS DIFFERENT PIECES — both carry a
|
||||
gated name, and they share none. An unnamed drop (a canvas screenshot)
|
||||
fits anywhere, which is the whole of the Marin case: two early stages both
|
||||
nearest to the same later one are one trickle, not an ambiguity.
|
||||
|
||||
What it does NOT refuse is a drop the creator made two pieces in
|
||||
themselves. Measured on artist 8: one November message carries both
|
||||
`AdL01_wip4` and `Year_20k_wip_z4`, so its drop holds both names, and a
|
||||
later drop of either piece joins it on its own name. That is the creator's
|
||||
co-posting carried forward — Discord shows those two together too — not a
|
||||
bridge FC built.
|
||||
"""
|
||||
named = [n for n in name_sets if n]
|
||||
return all(a & b for i, a in enumerate(named) for b in named[i + 1:])
|
||||
|
||||
|
||||
async def _merge_drop(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
into: _Drop,
|
||||
drop: _Drop,
|
||||
route: str,
|
||||
min_images: int,
|
||||
cooldown: timedelta,
|
||||
) -> None:
|
||||
"""Absorb `drop`'s messages into `into`, carry its links over, delete it.
|
||||
|
||||
Growth is stamped at the merged messages' OWN time, not the wall clock.
|
||||
Merging history must not drag a two-year-old drop to the top of the feed,
|
||||
and the time the group actually grew is when those messages arrived.
|
||||
"""
|
||||
await _repoint_associations(session, from_id=drop.post.id, to_id=into.post.id)
|
||||
grew_before = into.post.last_grew_at
|
||||
await _absorb_into(
|
||||
session, group=into.post, member_ids=sorted(drop.members),
|
||||
source_id=into.post.source_id, now=drop.last_at,
|
||||
min_images=min_images, cooldown=cooldown,
|
||||
)
|
||||
# Never backwards: a group that already grew later than these messages
|
||||
# keeps that later date.
|
||||
if grew_before is not None and grew_before > drop.last_at:
|
||||
into.post.last_grew_at = grew_before
|
||||
details = dict(into.post.synthesis_details or {})
|
||||
if grew_before is not None and grew_before > drop.last_at:
|
||||
details["last_grew_at"] = grew_before.isoformat()
|
||||
# The honesty rule, extended: a grouping FC invented says what it was
|
||||
# built from, and a merge says WHY — "name:svtt" or "nearest".
|
||||
details["merged"] = [
|
||||
*details.get("merged", []),
|
||||
{"post_id": drop.post.id, "route": route, "message_ids": sorted(drop.members)},
|
||||
]
|
||||
details["nearest_message_ids"] = sorted((into.nearest or set()) | (drop.nearest or set()))
|
||||
into.post.synthesis_details = details
|
||||
|
||||
into.members |= drop.members
|
||||
into.names |= drop.names
|
||||
into.images += drop.images
|
||||
into.nearest = (into.nearest or set()) | (drop.nearest or set())
|
||||
into.last_at = max(into.last_at, drop.last_at)
|
||||
|
||||
await session.execute(delete(ImageProvenance).where(ImageProvenance.post_id == drop.post.id))
|
||||
await session.delete(drop.post)
|
||||
await session.flush()
|
||||
|
||||
|
||||
async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||
"""Group every enabled Discord source. No-op when the switch is off.
|
||||
|
||||
@@ -612,6 +959,7 @@ async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||
if not settings.discord_grouping_enabled:
|
||||
return {
|
||||
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
||||
"drops_merged": 0,
|
||||
}
|
||||
|
||||
sources = (await session.execute(
|
||||
@@ -626,6 +974,7 @@ async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||
|
||||
created = 0
|
||||
joined = 0
|
||||
merged = 0
|
||||
for source in sources:
|
||||
joined += await join_open_groups(
|
||||
session, source,
|
||||
@@ -644,12 +993,20 @@ async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||
window_minutes=window_minutes,
|
||||
now=now,
|
||||
)
|
||||
# Last, so the drops the two passes above just wrote are merged in
|
||||
# the same sweep rather than showing as singletons for an hour.
|
||||
merged += await merge_trickles(
|
||||
session, source,
|
||||
gap=timedelta(hours=float(settings.discord_group_close_after_hours)),
|
||||
min_images=int(settings.discord_group_resurface_min_images),
|
||||
cooldown=timedelta(hours=float(settings.discord_group_resurface_cooldown_hours)),
|
||||
)
|
||||
log.info(
|
||||
"discord drop grouping: %d source(s), %d synthetic post(s) created, "
|
||||
"%d image(s) joined to open groups",
|
||||
len(sources), created, joined,
|
||||
"%d image(s) joined to open groups, %d trickle drop(s) merged",
|
||||
len(sources), created, joined, merged,
|
||||
)
|
||||
return {
|
||||
"enabled": True, "sources": len(sources),
|
||||
"posts_created": created, "images_joined": joined,
|
||||
"posts_created": created, "images_joined": joined, "drops_merged": merged,
|
||||
}
|
||||
|
||||
@@ -165,6 +165,19 @@ MAX_TOKEN_POSTS = 6
|
||||
# four-post band, where conto's `illustration9` and `maid` also sit.
|
||||
IDENTITY_FLOOR = 0.75
|
||||
|
||||
# A LEADING name spanning this many of ONE ARTIST's posts is a habit — a
|
||||
# character the creator returns to — not one piece's trickle. Used wherever a
|
||||
# name gathers a FAMILY: the teaser card's variants (#4401) and the Discord
|
||||
# grouper's trickle merge (#4390), so the two cannot disagree about what a
|
||||
# family is.
|
||||
#
|
||||
# Its own value rather than MAX_TOKEN_POSTS (6), which is calibrated for
|
||||
# PAIRING two posts and measured too tight for a family. On artist 8,
|
||||
# `tentacooler` spans 6 posts over 7 days and `0-k1` 6 posts over 10: both real
|
||||
# families, both gated out at 6. At 8, `anya` (7) passes the cap — and has no
|
||||
# pair inside any family window, which is what the window is for.
|
||||
FAMILY_MAX_POSTS = 8
|
||||
|
||||
|
||||
def _strip_prefixes(stem: str) -> str:
|
||||
"""Remove the framing each platform's importer adds around the real name."""
|
||||
|
||||
@@ -58,17 +58,7 @@ from ..utils.text import html_to_plain, truncate_at_word
|
||||
from .discord_grouping import PLATFORM as DISCORD
|
||||
from .gallery_service import thumbnail_url
|
||||
from .post_association_service import DUPLICATE_MAX_DISTANCE
|
||||
from .post_naming import leading_name, rarity, token_frequencies
|
||||
|
||||
# A leading name spanning this many of ONE ARTIST's posts is a habit — a
|
||||
# character the creator returns to — not one piece's trickle.
|
||||
#
|
||||
# Its own value rather than post_naming.MAX_TOKEN_POSTS (6), which is
|
||||
# calibrated for PAIRING two posts, and measured too tight for a family. On
|
||||
# artist 8, `tentacooler` spans 6 posts over 7 days and `0-k1` 6 posts over 10:
|
||||
# both real families, both gated out at 6. At 8, `anya` (7) passes the cap —
|
||||
# and has no pair inside the family window, which is what the window is for.
|
||||
FAMILY_MAX_POSTS = 8
|
||||
from .post_naming import FAMILY_MAX_POSTS, leading_name, rarity, token_frequencies
|
||||
|
||||
# The text each referenced post contributes to the card, per post. The card
|
||||
# clamps it again; this keeps a long Discord thread from making the feed
|
||||
|
||||
@@ -1175,7 +1175,7 @@ def group_discord_drops() -> str:
|
||||
return "disabled"
|
||||
return (
|
||||
f"sources={res['sources']} created={res['posts_created']} "
|
||||
f"joined={res['images_joined']}"
|
||||
f"joined={res['images_joined']} merged={res['drops_merged']}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -374,6 +374,7 @@ async def test_the_sweep_is_a_no_op_when_the_switch_is_off(db):
|
||||
# `images_joined` broke this assertion, which is exactly what it is for.
|
||||
assert result == {
|
||||
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
||||
"drops_merged": 0,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
"""#4390: a creator's trickle is one drop, not a card per message.
|
||||
|
||||
Operator, 2026-09-24, on a feed of "Grouped from 1 Discord message" cards:
|
||||
*"the groups are still single image even when they can clearly be seen as
|
||||
group"*. 665 of Yellowroom's 714 drops were one message, because both join
|
||||
paths demand closeness to a drop's FIRST image and a piece's stages drift away
|
||||
from it — each is nearest the one before, not the first.
|
||||
|
||||
The merge pass joins a later drop to an earlier one within 7 days when they
|
||||
share a working name, or when one's image is the other's nearest neighbour in
|
||||
the artist's library. Vectors here are built at stated angles (see
|
||||
test_discord_grouping._vec) so each test says exactly which image is nearest
|
||||
to which, rather than hoping.
|
||||
"""
|
||||
import math
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageRecord,
|
||||
Post,
|
||||
PostAssociation,
|
||||
Source,
|
||||
)
|
||||
from backend.app.services.discord_grouping import (
|
||||
DROP_GROUPER,
|
||||
_compatible,
|
||||
group_source,
|
||||
merge_trickles,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
DIM = 1152
|
||||
GAP = timedelta(hours=168)
|
||||
T0 = datetime.now(UTC) - timedelta(days=30)
|
||||
|
||||
|
||||
def _vec(angle: float) -> list[float]:
|
||||
v = [0.0] * DIM
|
||||
v[0] = math.cos(angle)
|
||||
v[1] = math.sin(angle)
|
||||
return v
|
||||
|
||||
|
||||
# --- the compatibility rule, pure --------------------------------------------
|
||||
|
||||
|
||||
def test_unnamed_stages_fit_with_anything():
|
||||
"""The Marin case: canvas screenshots carry no name, and two early stages
|
||||
both nearest to one later stage are one trickle, not an ambiguity."""
|
||||
assert _compatible([set(), set(), set()])
|
||||
assert _compatible([set(), {"svtt"}])
|
||||
|
||||
|
||||
def test_two_differently_named_pieces_never_meet():
|
||||
assert not _compatible([{"alpha"}, {"beta"}, set()])
|
||||
|
||||
|
||||
def test_pieces_sharing_a_name_are_one_piece():
|
||||
assert _compatible([{"year", "20k"}, {"year"}])
|
||||
|
||||
|
||||
# --- end to end --------------------------------------------------------------
|
||||
|
||||
|
||||
async def _seed(db, name):
|
||||
artist = Artist(name=name, slug=name)
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
source = Source(artist_id=artist.id, platform="discord",
|
||||
url=f"https://discord.com/channels/1/{name}", enabled=True)
|
||||
db.add(source)
|
||||
await db.flush()
|
||||
return artist, source
|
||||
|
||||
|
||||
_n = iter(range(1, 100_000))
|
||||
|
||||
|
||||
async def _message(db, artist, source, *, at, angle, name=None, text=None):
|
||||
n = next(_n)
|
||||
post = Post(source_id=source.id, artist_id=artist.id,
|
||||
external_post_id=f"msg-{n}", post_date=at, description=text)
|
||||
db.add(post)
|
||||
await db.flush()
|
||||
# gallery-dl's Discord shape, so the prefix strips and NAME is the working
|
||||
# name. Unnamed messages are canvas screenshots, which carry none.
|
||||
stem = name or f"Screenshot_2026-09-08_{n:06d}"
|
||||
db.add(ImageRecord(
|
||||
path=f"/images/{artist.slug}/20260901_{1234567890000 + n}_01_{stem}.png",
|
||||
sha256=f"{n:064d}", size_bytes=10, mime="image/png", width=10, height=10,
|
||||
origin="downloaded", primary_post_id=post.id, artist_id=artist.id,
|
||||
siglip_embedding=_vec(angle),
|
||||
))
|
||||
await db.flush()
|
||||
return post
|
||||
|
||||
|
||||
async def _group_then_merge(db, source):
|
||||
"""E2 makes the singletons exactly as it does live; then the merge pass."""
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
merged = await merge_trickles(
|
||||
db, source, gap=GAP, min_images=2, cooldown=timedelta(hours=24),
|
||||
)
|
||||
await db.commit()
|
||||
return merged
|
||||
|
||||
|
||||
async def _drops(db, source):
|
||||
return (await db.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == source.id,
|
||||
Post.synthesized_by == DROP_GROUPER,
|
||||
Post.absorbed_by_post_id.is_(None),
|
||||
).order_by(Post.post_date)
|
||||
)).scalars().all()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stages_a_day_apart_become_one_drop(db):
|
||||
"""The screenshot, measured shape: each stage 0.12 from the next and 0.46
|
||||
from the first, so seed distance splits them and nearest-neighbour joins
|
||||
them."""
|
||||
artist, source = await _seed(db, "trickle-artist")
|
||||
a = await _message(db, artist, source, at=T0, angle=0.0, text="Very early Marin.")
|
||||
b = await _message(db, artist, source, at=T0 + timedelta(hours=17), angle=0.5,
|
||||
text="Might get mirrored.")
|
||||
c = await _message(db, artist, source, at=T0 + timedelta(hours=20), angle=1.0,
|
||||
text="Got there eventually.")
|
||||
await db.commit()
|
||||
|
||||
assert await _group_then_merge(db, source) == 2
|
||||
|
||||
(drop,) = await _drops(db, source)
|
||||
assert set(drop.synthesis_details["member_post_ids"]) == {a.id, b.id, c.id}
|
||||
assert drop.synthesis_details["message_count"] == 3
|
||||
# The body is every message's text, in arrival order.
|
||||
assert drop.description.index("Very early") < drop.description.index("Got there")
|
||||
# And it says why — a grouping FC invented has to be checkable.
|
||||
assert {m["route"] for m in drop.synthesis_details["merged"]} == {"nearest"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_shared_working_name_joins_across_days(db):
|
||||
"""`svtt_wip3` did not join `svtt_wip4` from the day before on the live
|
||||
instance. Orthogonal vectors here, so only the name can do it — and the
|
||||
route it records says so."""
|
||||
artist, source = await _seed(db, "named-artist")
|
||||
await _message(db, artist, source, at=T0, angle=0.0, name="svtt_wip3")
|
||||
await _message(db, artist, source, at=T0 + timedelta(days=2), angle=math.pi / 2,
|
||||
name="svtt_drench_b")
|
||||
await db.commit()
|
||||
|
||||
await _group_then_merge(db, source)
|
||||
|
||||
(drop,) = await _drops(db, source)
|
||||
assert drop.synthesis_details["merged"][0]["route"] == "name:svtt"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nothing_joins_across_a_quiet_week(db):
|
||||
"""Measured: where the nearest neighbour was another message within 7 days
|
||||
the names agreed 53 times of 53; past 7 days they begin to disagree."""
|
||||
artist, source = await _seed(db, "quiet-artist")
|
||||
await _message(db, artist, source, at=T0, angle=0.0, name="alpha_wip1")
|
||||
await _message(db, artist, source, at=T0 + timedelta(days=8), angle=0.05,
|
||||
name="alpha_base")
|
||||
await db.commit()
|
||||
|
||||
assert await _group_then_merge(db, source) == 0
|
||||
assert len(await _drops(db, source)) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_named_pieces_meeting_through_a_third_stay_apart(db):
|
||||
"""C's image is nearest A's; B's image is nearest C's. A is `alpha`, B is
|
||||
`beta` — so C reaches two different pieces and nothing moves."""
|
||||
artist, source = await _seed(db, "bridge-artist")
|
||||
await _message(db, artist, source, at=T0, angle=0.0, name="alpha")
|
||||
await _message(db, artist, source, at=T0 + timedelta(hours=3), angle=0.6, name="beta")
|
||||
await _message(db, artist, source, at=T0 + timedelta(days=1), angle=0.25)
|
||||
await db.commit()
|
||||
|
||||
assert await _group_then_merge(db, source) == 0
|
||||
assert len(await _drops(db, source)) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_teaser_link_survives_its_drop_being_merged(db):
|
||||
"""The association's payload FK cascades. Merging the drop a teaser was
|
||||
linked to must carry the link across, or it silently undoes #4402."""
|
||||
artist, source = await _seed(db, "linked-artist")
|
||||
patreon = Source(artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/linked-artist", enabled=True)
|
||||
db.add(patreon)
|
||||
await db.flush()
|
||||
teaser = Post(source_id=patreon.id, artist_id=artist.id, external_post_id="teaser",
|
||||
post_date=T0 + timedelta(days=1, hours=2))
|
||||
db.add(teaser)
|
||||
await _message(db, artist, source, at=T0, angle=0.0, name="svtt_wip3")
|
||||
await _message(db, artist, source, at=T0 + timedelta(days=1), angle=math.pi / 2,
|
||||
name="svtt_drench_b")
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
later = (await _drops(db, source))[-1]
|
||||
db.add(PostAssociation(announcement_post_id=teaser.id, payload_post_id=later.id,
|
||||
score=1.0, status="linked", linked_by="fc"))
|
||||
await db.commit()
|
||||
|
||||
await merge_trickles(db, source, gap=GAP, min_images=2, cooldown=timedelta(hours=24))
|
||||
await db.commit()
|
||||
|
||||
(drop,) = await _drops(db, source)
|
||||
link = (await db.execute(select(PostAssociation))).scalar_one()
|
||||
assert (link.payload_post_id, link.status, link.linked_by) == (drop.id, "linked", "fc")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merging_history_does_not_drag_it_to_the_top_of_the_feed(db):
|
||||
"""Growth is stamped at the merged messages' own time. A merge the first
|
||||
sweep makes over two-year-old drops must not read as news today."""
|
||||
artist, source = await _seed(db, "history-artist")
|
||||
old = datetime.now(UTC) - timedelta(days=700)
|
||||
await _message(db, artist, source, at=old, angle=0.0, name="svtt_wip1")
|
||||
await _message(db, artist, source, at=old + timedelta(days=1), angle=0.5, name="svtt_wip2")
|
||||
await _message(db, artist, source, at=old + timedelta(days=2), angle=1.0, name="svtt_base")
|
||||
await db.commit()
|
||||
|
||||
await _group_then_merge(db, source)
|
||||
|
||||
(drop,) = await _drops(db, source)
|
||||
assert drop.last_grew_at <= old + timedelta(days=2, minutes=1)
|
||||
assert drop.resurfaced_at is None or drop.resurfaced_at <= old + timedelta(days=2, minutes=1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_checked_drop_is_not_examined_again(db):
|
||||
"""Each drop costs one nearest-neighbour query per image, once. The flag is
|
||||
what stops every hourly sweep repeating the whole history."""
|
||||
artist, source = await _seed(db, "checked-artist")
|
||||
await _message(db, artist, source, at=T0, angle=0.0, name="alpha")
|
||||
await _message(db, artist, source, at=T0 + timedelta(days=3), angle=math.pi / 2,
|
||||
name="beta")
|
||||
await db.commit()
|
||||
await _group_then_merge(db, source)
|
||||
|
||||
for drop in await _drops(db, source):
|
||||
assert drop.synthesis_details["trickle_checked"] is True
|
||||
assert "nearest_message_ids" in drop.synthesis_details
|
||||
@@ -29,11 +29,8 @@ from backend.app.models import (
|
||||
from backend.app.services.discord_grouping import DROP_GROUPER
|
||||
from backend.app.services.post_association_service import PostAssociationService
|
||||
from backend.app.services.post_feed_service import PostFeedService
|
||||
from backend.app.services.post_unification import (
|
||||
FAMILY_MAX_POSTS,
|
||||
Candidate,
|
||||
family,
|
||||
)
|
||||
from backend.app.services.post_naming import FAMILY_MAX_POSTS
|
||||
from backend.app.services.post_unification import Candidate, family
|
||||
|
||||
T0 = datetime(2026, 9, 1, 12, 0, tzinfo=UTC)
|
||||
WINDOW = timedelta(days=60)
|
||||
|
||||
Reference in New Issue
Block a user