What you pay for, what Discord drops, and pixiv switched off #251
@@ -0,0 +1,86 @@
|
||||
"""An open grouping — a synthetic post that a later drop can still join.
|
||||
|
||||
Milestone 388, step E3. E2's synthetic post was sealed at creation: a creator
|
||||
who added two more variants the next day started a second post. These two
|
||||
columns let the group stay open and absorb the follow-up, without the post
|
||||
either freezing or thrashing the feed.
|
||||
|
||||
## Why openness is derived rather than stored
|
||||
|
||||
There is no `closed_at` here on purpose. A group is open if it grew (or
|
||||
started) within `ml_settings.discord_group_close_after_hours`, so openness is a
|
||||
comparison rather than a state — which means lowering the setting closes old
|
||||
groups and raising it reopens them, with nothing to repair either way. A stored
|
||||
flag would need its own sweep to set it and its own repair path to ever change
|
||||
the policy, for no gain.
|
||||
|
||||
## Why `resurfaced_at` is separate from `last_grew_at`
|
||||
|
||||
They answer different questions. `last_grew_at` is when the group last
|
||||
absorbed something — it decides how long the group stays joinable and is what
|
||||
the card shows. `resurfaced_at` is the FEED POSITION, advanced only when the
|
||||
anti-thrash rule fires, so a group that gains one image a day updates in place
|
||||
while a genuine second wave moves once. Folding them together would make every
|
||||
addition a bump, which is the annoyance this step exists to avoid.
|
||||
|
||||
Both are NULL on every ordinary post, so the feed's sort key can COALESCE
|
||||
through `resurfaced_at` without moving anything that is not a grouping.
|
||||
|
||||
Revision ID: 0093
|
||||
Revises: 0092
|
||||
Create Date: 2026-09-10
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0093"
|
||||
down_revision: Union[str, None] = "0092"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"post", sa.Column("last_grew_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"post", sa.Column("resurfaced_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
# No index on either. The feed already sorts on an unindexed
|
||||
# COALESCE(post_date, downloaded_at) expression, so adding resurfaced_at to
|
||||
# that COALESCE changes nothing about how the query plans — and inventing a
|
||||
# functional index here would be guessing at the fix for a cost nobody has
|
||||
# measured. Measuring it is step B2's job.
|
||||
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_close_after_hours", sa.Float(),
|
||||
server_default=sa.text("168"), nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_resurface_min_images", sa.Integer(),
|
||||
server_default="2", nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_resurface_cooldown_hours", sa.Float(),
|
||||
server_default=sa.text("24"), nullable=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("ml_settings", "discord_group_resurface_cooldown_hours")
|
||||
op.drop_column("ml_settings", "discord_group_resurface_min_images")
|
||||
op.drop_column("ml_settings", "discord_group_close_after_hours")
|
||||
op.drop_column("post", "resurfaced_at")
|
||||
op.drop_column("post", "last_grew_at")
|
||||
@@ -54,6 +54,11 @@ _EDITABLE = (
|
||||
"discord_grouping_enabled",
|
||||
"discord_group_max_distance",
|
||||
"discord_group_window_minutes",
|
||||
# E3: how long a grouping stays open, and the anti-thrash rule that keeps
|
||||
# a growing one from monopolising the feed.
|
||||
"discord_group_close_after_hours",
|
||||
"discord_group_resurface_min_images",
|
||||
"discord_group_resurface_cooldown_hours",
|
||||
*_DETECTOR_FIELDS,
|
||||
)
|
||||
|
||||
@@ -162,6 +167,16 @@ def _validate(p: dict) -> str | None:
|
||||
return "discord_group_max_distance must be between 0 and 1"
|
||||
if float(p["discord_group_window_minutes"]) <= 0:
|
||||
return "discord_group_window_minutes must be > 0"
|
||||
# A group must stay open at least as long as the drop window it was cut
|
||||
# with, or the joiner could never reach a message the grouper deferred —
|
||||
# the two would fight, and the symptom (drops that never grow) would look
|
||||
# like the predicate failing rather than a settings contradiction.
|
||||
if float(p["discord_group_close_after_hours"]) * 60 < float(p["discord_group_window_minutes"]):
|
||||
return "discord_group_close_after_hours must be at least the drop window"
|
||||
if int(p["discord_group_resurface_min_images"]) < 1:
|
||||
return "discord_group_resurface_min_images must be >= 1"
|
||||
if float(p["discord_group_resurface_cooldown_hours"]) < 0:
|
||||
return "discord_group_resurface_cooldown_hours must be >= 0"
|
||||
# Embedder model swap (#1190): both must be non-empty. Changing them means a
|
||||
# different embedding space — the operator must re-embed + retrain after.
|
||||
for key in ("embedder_model_name", "embedder_model_version"):
|
||||
|
||||
@@ -286,6 +286,35 @@ class MLSettings(Base):
|
||||
Float, nullable=False, default=60.0,
|
||||
server_default=text("60"),
|
||||
)
|
||||
# How long a synthetic post keeps accepting new members (#388 E3). This is
|
||||
# NOT the drop window above: the window cuts one sweep's messages into
|
||||
# drops, this decides how long a finished drop can still be REJOINED when a
|
||||
# creator adds variants days later. A week by default — long enough for the
|
||||
# "and here is the alt outfit" follow-up that motivated the feature, short
|
||||
# enough that a group does not still be open when the same character comes
|
||||
# round again months later and gets absorbed by mistake.
|
||||
#
|
||||
# Openness is DERIVED from this, not stored: a group is open if it grew (or
|
||||
# started) within this period. So lowering it closes old groups and raising
|
||||
# it reopens them, which is comprehensible and reversible — the alternative,
|
||||
# a stored closed_at, would need its own repair path to ever change.
|
||||
discord_group_close_after_hours: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=168.0,
|
||||
server_default=text("168"),
|
||||
)
|
||||
# Anti-thrash (#388 E3). An updated post SHOULD be visible — that is the
|
||||
# point of keeping it open — but a group gaining one image a day must not
|
||||
# monopolise the feed. Growth smaller than this never moves the post, and
|
||||
# no group moves more than once per cooldown, so a drip-feed updates in
|
||||
# place while a real second wave resurfaces exactly once.
|
||||
discord_group_resurface_min_images: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=2,
|
||||
server_default="2",
|
||||
)
|
||||
discord_group_resurface_cooldown_hours: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=24.0,
|
||||
server_default=text("24"),
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -136,3 +136,27 @@ class Post(Base):
|
||||
absorbed_by_post_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
# -- An OPEN grouping (milestone 388 E3) -------------------------------
|
||||
# A synthetic post is not sealed at creation: a creator who adds two more
|
||||
# variants the next day extends the existing post rather than starting a
|
||||
# new one. These two columns are what make that possible without the post
|
||||
# either freezing or thrashing the feed.
|
||||
#
|
||||
# `last_grew_at` is when the group last absorbed something. It answers two
|
||||
# questions: how long the group stays JOINABLE (a group closes after a
|
||||
# quiet period — artists reuse characters for years, and a group left open
|
||||
# forever will eventually absorb something it shouldn't), and what the card
|
||||
# shows as "updated N ago". NULL means it has never grown since creation.
|
||||
last_grew_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
# The feed position, and ONLY set when the anti-thrash rule fires — see
|
||||
# discord_grouping.should_resurface. A group that gains one image a day
|
||||
# must not sit permanently at the top of the feed, so growth updates the
|
||||
# post without necessarily moving it; a genuine second wave moves it once.
|
||||
#
|
||||
# NULL on every ordinary post, which is why the feed's sort key can
|
||||
# COALESCE through it without changing where anything else lands.
|
||||
resurfaced_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
|
||||
@@ -245,6 +245,10 @@ async def _synthesize(
|
||||
"max_distance": max_distance,
|
||||
"window_minutes": window_minutes,
|
||||
"grouped_at": datetime.now(UTC).isoformat(),
|
||||
# Growth accumulated since the post last moved in the feed (E3).
|
||||
# Seeded here so the joiner never meets its absence — creation IS
|
||||
# a surfacing, so the count starts at zero.
|
||||
"images_since_surface": 0,
|
||||
},
|
||||
)
|
||||
session.add(post)
|
||||
@@ -256,31 +260,52 @@ async def _synthesize(
|
||||
.values(absorbed_by_post_id=post.id)
|
||||
)
|
||||
|
||||
# Link every member image to the synthetic post via provenance. The feed
|
||||
# and detail views already union provenance with primary_post_id
|
||||
# (post_feed_service._thumbnails_for), so this alone makes the drop's
|
||||
# images show up under the post FC wrote — no second render path.
|
||||
#
|
||||
# primary_post_id is deliberately NOT rewritten: the message post remains
|
||||
# the image's true origin, and the synthetic post is an ADDITIONAL claim on
|
||||
# it, which is what keeps the grouping reversible.
|
||||
image_rows = (await session.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.primary_post_id.in_([m.id for m in members]))
|
||||
)).scalars().all()
|
||||
if image_rows:
|
||||
await session.execute(
|
||||
pg_insert(ImageProvenance)
|
||||
.values([
|
||||
{"image_record_id": iid, "post_id": post.id, "source_id": source.id}
|
||||
for iid in image_rows
|
||||
])
|
||||
# (image, post) is unique; a re-run that raced itself is a no-op
|
||||
# rather than an IntegrityError that loses the whole sweep.
|
||||
.on_conflict_do_nothing(constraint="uq_image_provenance_image_post")
|
||||
)
|
||||
await _link_member_images(
|
||||
session, post_id=post.id, member_ids=[m.id for m in members],
|
||||
source_id=source.id,
|
||||
)
|
||||
return post
|
||||
|
||||
|
||||
async def _link_member_images(
|
||||
session: AsyncSession, *, post_id: int, member_ids: list[int], source_id: int | None,
|
||||
) -> int:
|
||||
"""Attach every member's images to the synthetic post. Returns how many.
|
||||
|
||||
The feed and detail views already union provenance with `primary_post_id`
|
||||
(post_feed_service._thumbnails_for), so this alone makes the drop's images
|
||||
show up under the post FC wrote — no second render path.
|
||||
|
||||
`primary_post_id` is deliberately NOT rewritten: the message post remains
|
||||
the image's true origin, and the synthetic post is an ADDITIONAL claim on
|
||||
it, which is what keeps the grouping reversible.
|
||||
|
||||
Shared by creation and by the E3 joiner rather than written twice, because
|
||||
the two would otherwise be free to drift on exactly the detail (which post
|
||||
owns the image) that makes a grouping reversible.
|
||||
"""
|
||||
if not member_ids:
|
||||
return 0
|
||||
image_rows = (await session.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids))
|
||||
)).scalars().all()
|
||||
if not image_rows:
|
||||
return 0
|
||||
await session.execute(
|
||||
pg_insert(ImageProvenance)
|
||||
.values([
|
||||
{"image_record_id": iid, "post_id": post_id, "source_id": source_id}
|
||||
for iid in image_rows
|
||||
])
|
||||
# (image, post) is unique. This is a BACKSTOP against a re-run that
|
||||
# raced itself, not the correctness argument: callers only ever pass
|
||||
# members that were unabsorbed a moment ago, so a conflict here means
|
||||
# concurrency, not a logic error.
|
||||
.on_conflict_do_nothing(constraint="uq_image_provenance_image_post")
|
||||
)
|
||||
return len(image_rows)
|
||||
|
||||
|
||||
async def group_source(
|
||||
session: AsyncSession,
|
||||
source: Source,
|
||||
@@ -329,11 +354,265 @@ async def group_source(
|
||||
return created
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# E3: an open grouping — a later drop joins its post and updates it.
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# A synthetic post is not sealed at creation. A creator who adds two more
|
||||
# variants the next day extends the existing post rather than starting a new
|
||||
# one, and its body grows with the new messages. That is what makes chat
|
||||
# capture read as content TRICKLING IN rather than as a stream of separate
|
||||
# arrivals.
|
||||
#
|
||||
# Three hard problems, each answered deliberately below: bridging (a candidate
|
||||
# near two groups), re-surfacing without thrashing the feed, and groups that
|
||||
# stay open forever.
|
||||
|
||||
# How much closer the nearest group must be than the runner-up before a
|
||||
# candidate is assigned to it at all.
|
||||
#
|
||||
# NOT a setting, deliberately. It is not a quality dial the operator would tune
|
||||
# toward a better feed — it expresses "these two are too close to call", and
|
||||
# exposing it would invite turning it to zero, which is precisely the silent
|
||||
# arbitrary choice it exists to prevent. When a candidate is genuinely between
|
||||
# two groups the recoverable answer is to leave it out and let it start its
|
||||
# own; the unrecoverable one is to merge, because a merge rewrites history —
|
||||
# two posts the operator may already have seen become one, and anything
|
||||
# pointing at the absorbed post dangles.
|
||||
AMBIGUITY_MARGIN = 0.02
|
||||
|
||||
|
||||
def should_resurface(
|
||||
*,
|
||||
images_since_surface: int,
|
||||
last_surface_at: datetime,
|
||||
now: datetime,
|
||||
min_images: int,
|
||||
cooldown: timedelta,
|
||||
) -> bool:
|
||||
"""Has this group grown enough, and waited long enough, to move in the feed?
|
||||
|
||||
An updated post SHOULD be visible — that is the point of keeping it open —
|
||||
but a group gaining one image a day must not sit permanently at the top.
|
||||
Both conditions have to hold: enough new images that the update is worth an
|
||||
interruption, and enough time since the last one that a steady drip cannot
|
||||
chain bumps together.
|
||||
"""
|
||||
if images_since_surface < min_images:
|
||||
return False
|
||||
return now - last_surface_at >= cooldown
|
||||
|
||||
|
||||
async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None:
|
||||
"""The embedding a group is measured against — its FIRST member's image.
|
||||
|
||||
Derived rather than stored, and derived by the same definition `build_groups`
|
||||
used (earliest member, lowest-id embedded image). Storing it at creation
|
||||
would have meant a backfill for groups already written and two definitions
|
||||
free to disagree; this way there is one.
|
||||
"""
|
||||
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||
return (await session.execute(
|
||||
select(ImageRecord.siglip_embedding)
|
||||
.join(Post, ImageRecord.primary_post_id == Post.id)
|
||||
.where(
|
||||
Post.absorbed_by_post_id == post_id,
|
||||
ImageRecord.siglip_embedding.is_not(None),
|
||||
)
|
||||
.order_by(sort_key, Post.id, ImageRecord.id)
|
||||
.limit(1)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
|
||||
async def open_groups(
|
||||
session: AsyncSession, source_id: int, *, now: datetime, close_after: timedelta,
|
||||
) -> list[tuple[Post, list[float]]]:
|
||||
"""This source's synthetic posts that are still accepting members.
|
||||
|
||||
Openness is DERIVED, not stored: a group is open if it grew — or started —
|
||||
within `close_after`. A group left open forever would eventually absorb
|
||||
something it shouldn't, because artists reuse characters for years; a
|
||||
stored `closed_at` would need a sweep to set it and a repair path to ever
|
||||
change the policy. This way the policy IS the query.
|
||||
"""
|
||||
cutoff = now - close_after
|
||||
posts = (await session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == source_id,
|
||||
Post.synthesized_by == DROP_GROUPER,
|
||||
# A synthetic post that was itself absorbed is not a thing today
|
||||
# (nothing absorbs one), but joining into one would nest groups.
|
||||
Post.absorbed_by_post_id.is_(None),
|
||||
func.coalesce(Post.last_grew_at, Post.post_date, Post.downloaded_at) >= cutoff,
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
out: list[tuple[Post, list[float]]] = []
|
||||
for post in posts:
|
||||
seed = await _group_seed(session, post.id)
|
||||
if seed is not None:
|
||||
out.append((post, seed))
|
||||
return out
|
||||
|
||||
|
||||
def assign_to_group(
|
||||
embedding: list[float],
|
||||
groups: list[tuple[Post, list[float]]],
|
||||
*,
|
||||
max_distance: float,
|
||||
) -> Post | None:
|
||||
"""Pick the one group this image belongs to, or None to leave it alone.
|
||||
|
||||
Returns None in two cases that mean different things and are deliberately
|
||||
treated the same: nothing is close enough (so E2 will start a new group
|
||||
from it), or two groups are BOTH close and too near each other to choose
|
||||
between (so E2 will start a new group from it). The second is the bridging
|
||||
case, and letting it start its own post is the recoverable failure —
|
||||
merging two existing posts is not.
|
||||
"""
|
||||
# key= on the distance ALONE. Sorting bare tuples falls through to the
|
||||
# second element when two distances tie, and `Post` has no ordering — so a
|
||||
# perfectly symmetric bridge (the exact case this function exists for)
|
||||
# would raise TypeError instead of declining to choose.
|
||||
scored = sorted(
|
||||
((cosine_distance(seed, embedding), post) for post, seed in groups),
|
||||
key=lambda pair: pair[0],
|
||||
)
|
||||
within = [(d, p) for d, p in scored if d <= max_distance]
|
||||
if not within:
|
||||
return None
|
||||
if len(within) >= 2 and (within[1][0] - within[0][0]) < AMBIGUITY_MARGIN:
|
||||
return None
|
||||
return within[0][1]
|
||||
|
||||
|
||||
async def _absorb_into(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
group: Post,
|
||||
member_ids: list[int],
|
||||
source_id: int | None,
|
||||
now: datetime,
|
||||
min_images: int,
|
||||
cooldown: timedelta,
|
||||
) -> int:
|
||||
"""Extend an existing synthetic post with new members. Returns images added."""
|
||||
members = (await session.execute(
|
||||
select(Post)
|
||||
.where(Post.id.in_(member_ids))
|
||||
.order_by(func.coalesce(Post.post_date, Post.downloaded_at), Post.id)
|
||||
)).scalars().all()
|
||||
if not members:
|
||||
return 0
|
||||
|
||||
added_images = await _link_member_images(
|
||||
session, post_id=group.id, member_ids=[m.id for m in members],
|
||||
source_id=source_id,
|
||||
)
|
||||
await session.execute(
|
||||
update(Post)
|
||||
.where(Post.id.in_([m.id for m in members]))
|
||||
.values(absorbed_by_post_id=group.id)
|
||||
)
|
||||
|
||||
# The new messages' text joins the body, in arrival order, exactly as at
|
||||
# creation — the post's body is the drop's text and the drop just grew.
|
||||
new_text = "\n\n".join(
|
||||
m.description.strip() for m in members if m.description and m.description.strip()
|
||||
)
|
||||
if new_text:
|
||||
group.description = f"{group.description}\n\n{new_text}" if group.description else new_text
|
||||
|
||||
details = dict(group.synthesis_details or {})
|
||||
existing_ids = list(details.get("member_post_ids") or [])
|
||||
details["member_post_ids"] = existing_ids + [
|
||||
m.id for m in members if m.id not in existing_ids
|
||||
]
|
||||
details["message_count"] = len(details["member_post_ids"])
|
||||
since = int(details.get("images_since_surface") or 0) + added_images
|
||||
details["last_grew_at"] = now.isoformat()
|
||||
|
||||
# The feed position moves only when the anti-thrash rule fires. Measured
|
||||
# from the last time the post actually MOVED (resurfaced_at), falling back
|
||||
# to when the drop started — creation is itself a surfacing.
|
||||
last_surface = group.resurfaced_at or group.post_date or group.downloaded_at
|
||||
if should_resurface(
|
||||
images_since_surface=since, last_surface_at=last_surface, now=now,
|
||||
min_images=min_images, cooldown=cooldown,
|
||||
):
|
||||
group.resurfaced_at = now
|
||||
since = 0
|
||||
details["images_since_surface"] = since
|
||||
|
||||
group.synthesis_details = details
|
||||
group.last_grew_at = now
|
||||
return added_images
|
||||
|
||||
|
||||
async def join_open_groups(
|
||||
session: AsyncSession,
|
||||
source: Source,
|
||||
*,
|
||||
max_distance: float,
|
||||
window_minutes: float,
|
||||
close_after_hours: float,
|
||||
resurface_min_images: int,
|
||||
resurface_cooldown_hours: float,
|
||||
now: datetime | None = None,
|
||||
) -> int:
|
||||
"""Offer this source's ungrouped messages to its open groups.
|
||||
|
||||
Runs BEFORE `group_source` in the sweep: a message that belongs to an
|
||||
existing drop must join it rather than found a rival post, and whichever
|
||||
runs first wins that message.
|
||||
"""
|
||||
now = now or datetime.now(UTC)
|
||||
window = timedelta(minutes=window_minutes)
|
||||
groups = await open_groups(
|
||||
session, source.id, now=now, close_after=timedelta(hours=close_after_hours),
|
||||
)
|
||||
if not groups:
|
||||
return 0
|
||||
|
||||
# Same quarantine as E2: a drop still arriving is left for the next run.
|
||||
rows = (await session.execute(
|
||||
_candidate_stmt(source.id, not_after=now - window)
|
||||
.limit(MAX_CANDIDATES_PER_SOURCE)
|
||||
)).all()
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
claimed: dict[int, list[int]] = {}
|
||||
for post_id, _occurred_at, embedding in rows:
|
||||
target = assign_to_group(embedding, groups, max_distance=max_distance)
|
||||
if target is not None:
|
||||
claimed.setdefault(target.id, []).append(post_id)
|
||||
|
||||
by_id = {post.id: post for post, _seed in groups}
|
||||
joined = 0
|
||||
for group_id, member_ids in claimed.items():
|
||||
joined += await _absorb_into(
|
||||
session, group=by_id[group_id], member_ids=member_ids,
|
||||
source_id=source.id, now=now,
|
||||
min_images=resurface_min_images,
|
||||
cooldown=timedelta(hours=resurface_cooldown_hours),
|
||||
)
|
||||
return joined
|
||||
|
||||
|
||||
async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||
"""Group every enabled Discord source. No-op when the switch is off."""
|
||||
"""Group every enabled Discord source. No-op when the switch is off.
|
||||
|
||||
Two passes per source, and the ORDER is load-bearing: offer new messages to
|
||||
the groups that are still open (E3) BEFORE founding new ones (E2). Whichever
|
||||
runs first claims a message, and a variant that belongs to yesterday's drop
|
||||
must extend that post rather than found a rival to it.
|
||||
"""
|
||||
settings = await MLSettings.load(session)
|
||||
if not settings.discord_grouping_enabled:
|
||||
return {"enabled": False, "sources": 0, "posts_created": 0}
|
||||
return {
|
||||
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
||||
}
|
||||
|
||||
sources = (await session.execute(
|
||||
select(Source).where(
|
||||
@@ -342,16 +621,35 @@ async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
max_distance = float(settings.discord_group_max_distance)
|
||||
window_minutes = float(settings.discord_group_window_minutes)
|
||||
|
||||
created = 0
|
||||
joined = 0
|
||||
for source in sources:
|
||||
joined += await join_open_groups(
|
||||
session, source,
|
||||
max_distance=max_distance,
|
||||
window_minutes=window_minutes,
|
||||
close_after_hours=float(settings.discord_group_close_after_hours),
|
||||
resurface_min_images=int(settings.discord_group_resurface_min_images),
|
||||
resurface_cooldown_hours=float(
|
||||
settings.discord_group_resurface_cooldown_hours
|
||||
),
|
||||
now=now,
|
||||
)
|
||||
created += await group_source(
|
||||
session, source,
|
||||
max_distance=float(settings.discord_group_max_distance),
|
||||
window_minutes=float(settings.discord_group_window_minutes),
|
||||
max_distance=max_distance,
|
||||
window_minutes=window_minutes,
|
||||
now=now,
|
||||
)
|
||||
log.info(
|
||||
"discord drop grouping: %d source(s), %d synthetic post(s) created",
|
||||
len(sources), created,
|
||||
"discord drop grouping: %d source(s), %d synthetic post(s) created, "
|
||||
"%d image(s) joined to open groups",
|
||||
len(sources), created, joined,
|
||||
)
|
||||
return {"enabled": True, "sources": len(sources), "posts_created": created}
|
||||
return {
|
||||
"enabled": True, "sources": len(sources),
|
||||
"posts_created": created, "images_joined": joined,
|
||||
}
|
||||
|
||||
@@ -41,8 +41,40 @@ THUMBNAIL_LIMIT = 6
|
||||
|
||||
|
||||
def _sort_key():
|
||||
"""Postgres COALESCE expression used in ORDER BY and WHERE clauses."""
|
||||
return func.coalesce(Post.post_date, Post.downloaded_at)
|
||||
"""Postgres COALESCE expression used in ORDER BY and WHERE clauses.
|
||||
|
||||
`resurfaced_at` leads (milestone 388 E3). A synthetic post stays OPEN — a
|
||||
creator who adds variants the next day extends the existing post — so such
|
||||
a post has two dates, and which one orders the feed is a real decision:
|
||||
|
||||
* ordering by when the drop STARTED buries a group that grows a week later
|
||||
under a week of other posts, so the operator never sees the new content —
|
||||
which defeats keeping the group open at all;
|
||||
* ordering by every growth lets a group that gains one image a day sit
|
||||
permanently at the top, so chat out-competes authored posts for the front
|
||||
page — the opposite of "post pacing stays front and centre".
|
||||
|
||||
So the feed orders by neither directly. `resurfaced_at` moves only when the
|
||||
anti-thrash rule fires (discord_grouping.should_resurface: enough new
|
||||
images AND enough time since the last move), which means a drip-feed
|
||||
updates IN PLACE and a genuine second wave resurfaces exactly once.
|
||||
|
||||
It is NULL on every ordinary post, so this COALESCE cannot move anything
|
||||
that is not a grouping. Used identically in ORDER BY and in the cursor's
|
||||
WHERE, which is what keeps pagination stable across the change.
|
||||
"""
|
||||
return func.coalesce(Post.resurfaced_at, Post.post_date, Post.downloaded_at)
|
||||
|
||||
|
||||
def _post_sort_value(post: Post):
|
||||
"""The Python twin of `_sort_key()`, for building a cursor from a loaded row.
|
||||
|
||||
Kept next to it on purpose: these two are one expression in two languages,
|
||||
and the failure when they disagree is not an error but a quiet one — rows
|
||||
skipped or repeated at page boundaries, which reads as a backend bug
|
||||
anywhere but here.
|
||||
"""
|
||||
return post.resurfaced_at or post.post_date or post.downloaded_at
|
||||
|
||||
|
||||
class PostFeedService:
|
||||
@@ -134,7 +166,10 @@ class PostFeedService:
|
||||
# Far edge in the travel direction: oldest row going older,
|
||||
# newest row going newer (rows is descending for display).
|
||||
edge_post = rows[-1][0] if direction == "older" else rows[0][0]
|
||||
edge_key = edge_post.post_date or edge_post.downloaded_at
|
||||
# Must match _sort_key() exactly, including resurfaced_at's
|
||||
# precedence: a cursor built from a different expression than the
|
||||
# ORDER BY silently skips or repeats rows at every page boundary.
|
||||
edge_key = _post_sort_value(edge_post)
|
||||
next_cursor = encode_cursor(edge_key, edge_post.id)
|
||||
|
||||
post_ids = [p.id for p, _, _ in rows]
|
||||
@@ -168,7 +203,7 @@ class PostFeedService:
|
||||
if anchor is None:
|
||||
return None
|
||||
anchor_post, anchor_artist, anchor_source = anchor
|
||||
anchor_key = anchor_post.post_date or anchor_post.downloaded_at
|
||||
anchor_key = _post_sort_value(anchor_post)
|
||||
anchor_cursor = encode_cursor(anchor_key, anchor_post.id)
|
||||
|
||||
older = await self.scroll(
|
||||
@@ -414,6 +449,10 @@ class PostFeedService:
|
||||
# keys are always present so the frontend never branches on absence.
|
||||
"synthesized_by": post.synthesized_by,
|
||||
"synthesis": post.synthesis_details,
|
||||
# #388 E3. A grouping stays open, so the card can say "updated N
|
||||
# ago" — which is the whole signal that chat content is trickling
|
||||
# in. NULL means it has not grown since it was created.
|
||||
"last_grew_at": post.last_grew_at.isoformat() if post.last_grew_at else None,
|
||||
# Non-null on a chat message a synthetic post absorbed. The feed
|
||||
# filters these out, but `around`/`get_post` still reach them, and
|
||||
# the UI uses this to explain why a post it linked to is not in the
|
||||
|
||||
@@ -1167,4 +1167,7 @@ def group_discord_drops() -> str:
|
||||
res = asyncio.run(_run())
|
||||
if not res["enabled"]:
|
||||
return "disabled"
|
||||
return f"sources={res['sources']} created={res['posts_created']}"
|
||||
return (
|
||||
f"sources={res['sources']} created={res['posts_created']} "
|
||||
f"joined={res['images_joined']}"
|
||||
)
|
||||
|
||||
@@ -27,6 +27,12 @@
|
||||
<span v-if="totalImages" class="fc-post-card__meta">
|
||||
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
||||
</span>
|
||||
<!-- Only on a grouping that has actually grown. An ordinary post can
|
||||
never show this, and a grouping that has not grown says nothing —
|
||||
an "updated" label that is always present teaches you to ignore it. -->
|
||||
<span v-if="grewAt" class="fc-post-card__meta fc-post-card__grew">
|
||||
· updated {{ grewRelative }}
|
||||
</span>
|
||||
<v-spacer />
|
||||
<PostSeriesMenu :post="post" />
|
||||
<v-btn
|
||||
@@ -221,16 +227,25 @@ const moreCount = computed(() => {
|
||||
const railCols = computed(() => rail.value.length + (moreCount.value > 0 ? 1 : 0))
|
||||
|
||||
const sortDateIso = computed(() => props.post.post_date || props.post.downloaded_at)
|
||||
|
||||
// #388 E3. A grouping stays OPEN, so its own date and its latest activity are
|
||||
// different facts. The card keeps showing when the drop STARTED — that is the
|
||||
// post's identity — and reports growth separately, because "this post is from
|
||||
// Tuesday but gained images this morning" is the whole signal that chat
|
||||
// content is trickling in.
|
||||
const grewAt = computed(() => (synthesized.value ? props.post.last_grew_at : null))
|
||||
const grewRelative = computed(() => (grewAt.value ? relativeFrom(grewAt.value) : ''))
|
||||
const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString())
|
||||
const relativeDate = computed(() => {
|
||||
const then = new Date(sortDateIso.value).getTime()
|
||||
function relativeFrom (iso) {
|
||||
const then = new Date(iso).getTime()
|
||||
const diff = (Date.now() - then) / 1000
|
||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||
if (diff < 86400 * 30) return `${Math.floor(diff / 86400)}d ago`
|
||||
return new Date(sortDateIso.value).toLocaleDateString()
|
||||
})
|
||||
return new Date(iso).toLocaleDateString()
|
||||
}
|
||||
const relativeDate = computed(() => relativeFrom(sortDateIso.value))
|
||||
|
||||
// --- images → post-scoped modal ---------------------------------------
|
||||
async function fullImageIds () {
|
||||
@@ -375,6 +390,10 @@ function formatBytes (n) {
|
||||
.fc-post-card__synthetic {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
|
||||
/* Growth is news, so it gets the accent the rest of the meta line doesn't —
|
||||
but it is still the meta line, not a badge competing with the artwork. */
|
||||
.fc-post-card__grew { color: rgb(var(--v-theme-accent)); }
|
||||
.fc-post-card__date,
|
||||
.fc-post-card__meta { white-space: nowrap; }
|
||||
|
||||
|
||||
@@ -48,6 +48,54 @@
|
||||
</div>
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<!-- E3: a grouping stays open, and these decide for how long and how
|
||||
loudly it announces that it grew. -->
|
||||
<div class="text-caption fc-muted mt-4 mb-2">
|
||||
A grouped post stays <strong>open</strong>: variants the creator adds
|
||||
later join the existing post instead of starting a new one, and its
|
||||
text grows with them.
|
||||
</div>
|
||||
<v-row>
|
||||
<v-col cols="12" sm="4">
|
||||
<SettingNumberField
|
||||
v-model="local.discord_group_close_after_hours"
|
||||
label="Stays open for (hours)" :min="1" :step="24"
|
||||
density="comfortable" max-width="none"
|
||||
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||
/>
|
||||
<div class="text-caption fc-muted mt-1">
|
||||
How long after its last addition a post still accepts new variants.
|
||||
Not the drop window above — that cuts one session into drops; this
|
||||
decides how late a follow-up can still join. Too long and the same
|
||||
character coming round again months later gets absorbed by mistake.
|
||||
</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="4">
|
||||
<SettingNumberField
|
||||
v-model="local.discord_group_resurface_min_images"
|
||||
label="New images before it resurfaces" :min="1" :step="1"
|
||||
density="comfortable" max-width="none"
|
||||
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||
/>
|
||||
<div class="text-caption fc-muted mt-1">
|
||||
Growth smaller than this updates the post where it sits instead of
|
||||
moving it back to the top of the feed.
|
||||
</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="4">
|
||||
<SettingNumberField
|
||||
v-model="local.discord_group_resurface_cooldown_hours"
|
||||
label="Resurface at most every (hours)" :min="0" :step="1"
|
||||
density="comfortable" max-width="none"
|
||||
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||
/>
|
||||
<div class="text-caption fc-muted mt-1">
|
||||
Together with the count above, this is what stops a post that gains
|
||||
an image a day from living permanently at the top of the feed.
|
||||
</div>
|
||||
</v-col>
|
||||
</v-row>
|
||||
</div>
|
||||
<div v-else><v-skeleton-loader type="paragraph" /></div>
|
||||
</MaintenanceTile>
|
||||
@@ -77,6 +125,10 @@ function onSave() {
|
||||
discord_grouping_enabled: Boolean(local.discord_grouping_enabled),
|
||||
discord_group_max_distance: Number(local.discord_group_max_distance),
|
||||
discord_group_window_minutes: Number(local.discord_group_window_minutes),
|
||||
discord_group_close_after_hours: Number(local.discord_group_close_after_hours),
|
||||
discord_group_resurface_min_images: Number(local.discord_group_resurface_min_images),
|
||||
discord_group_resurface_cooldown_hours:
|
||||
Number(local.discord_group_resurface_cooldown_hours),
|
||||
})
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -84,6 +84,36 @@ describe('PostCard', () => {
|
||||
expect(w.find('.fc-post-card__synthetic').exists()).toBe(false)
|
||||
})
|
||||
|
||||
it('reports growth separately from the post date', () => {
|
||||
// The drop's own date is its identity; growth is news about it. "From
|
||||
// Tuesday, gained images this morning" is the trickling-in signal, and
|
||||
// collapsing the two would erase it.
|
||||
const w = mountComponent(PostCard, {
|
||||
props: {
|
||||
post: { ...SYNTH, last_grew_at: new Date(Date.now() - 3600e3).toISOString() },
|
||||
},
|
||||
pinia: freshPinia(),
|
||||
})
|
||||
expect(w.text()).toContain('updated 1h ago')
|
||||
})
|
||||
|
||||
it('says nothing about growth on a grouping that has not grown', () => {
|
||||
// An "updated" label that is always there teaches you to ignore it.
|
||||
const w = mountComponent(PostCard, {
|
||||
props: { post: { ...SYNTH, last_grew_at: null } },
|
||||
pinia: freshPinia(),
|
||||
})
|
||||
expect(w.text()).not.toContain('updated')
|
||||
})
|
||||
|
||||
it('never claims an ordinary post grew, even if the field leaks in', () => {
|
||||
const w = mountComponent(PostCard, {
|
||||
props: { post: { ...BASE, last_grew_at: new Date().toISOString() } },
|
||||
pinia: freshPinia(),
|
||||
})
|
||||
expect(w.text()).not.toContain('updated')
|
||||
})
|
||||
|
||||
it('singularises a one-message drop', () => {
|
||||
const w = mountComponent(PostCard, {
|
||||
props: { post: { ...SYNTH, synthesis: { message_count: 1 } } },
|
||||
|
||||
@@ -16,9 +16,12 @@ from sqlalchemy import select
|
||||
from backend.app.models import Artist, ImageProvenance, ImageRecord, MLSettings, Post, Source
|
||||
from backend.app.services.discord_grouping import (
|
||||
DROP_GROUPER,
|
||||
assign_to_group,
|
||||
build_groups,
|
||||
cosine_distance,
|
||||
group_source,
|
||||
join_open_groups,
|
||||
should_resurface,
|
||||
sweep,
|
||||
)
|
||||
from backend.app.services.post_feed_service import PostFeedService
|
||||
@@ -387,3 +390,341 @@ async def test_the_sweep_only_touches_discord_sources(db):
|
||||
db.expunge_all()
|
||||
assert (await db.get(Post, a.id)).absorbed_by_post_id is None
|
||||
assert (await db.get(Post, b.id)).absorbed_by_post_id is None
|
||||
|
||||
|
||||
# --- E3: an open grouping -------------------------------------------------
|
||||
#
|
||||
# Three hard problems, one test class each. The anti-thrash rule is ASSERTED
|
||||
# rather than assumed, because "a group that grows daily monopolises the feed"
|
||||
# is the kind of defect nobody notices until they have lived with it.
|
||||
|
||||
|
||||
def test_growth_below_the_minimum_never_moves_the_post():
|
||||
"""One image a day must not chain bumps together."""
|
||||
now = datetime.now(UTC)
|
||||
assert not should_resurface(
|
||||
images_since_surface=1, last_surface_at=now - timedelta(days=30),
|
||||
now=now, min_images=2, cooldown=timedelta(hours=24),
|
||||
)
|
||||
|
||||
|
||||
def test_growth_inside_the_cooldown_never_moves_the_post():
|
||||
"""Even a big second wave waits — otherwise a burst bumps once per sweep."""
|
||||
now = datetime.now(UTC)
|
||||
assert not should_resurface(
|
||||
images_since_surface=10, last_surface_at=now - timedelta(hours=1),
|
||||
now=now, min_images=2, cooldown=timedelta(hours=24),
|
||||
)
|
||||
|
||||
|
||||
def test_a_real_second_wave_moves_the_post_once():
|
||||
now = datetime.now(UTC)
|
||||
assert should_resurface(
|
||||
images_since_surface=3, last_surface_at=now - timedelta(days=2),
|
||||
now=now, min_images=2, cooldown=timedelta(hours=24),
|
||||
)
|
||||
|
||||
|
||||
def _group(post_id: int, angle: float):
|
||||
return (Post(id=post_id), _vec(angle))
|
||||
|
||||
|
||||
def test_a_candidate_near_two_groups_joins_neither():
|
||||
"""Bridging. Merging would rewrite history — two posts the operator may
|
||||
already have seen become one — so the recoverable answer is to leave it and
|
||||
let it found its own group."""
|
||||
left, right = _group(1, -0.30), _group(2, 0.30)
|
||||
assert assign_to_group(_vec(0.0), [left, right], max_distance=0.5) is None
|
||||
|
||||
|
||||
def test_a_candidate_clearly_nearer_one_group_joins_it():
|
||||
near, far = _group(1, 0.02), _group(2, 1.2)
|
||||
chosen = assign_to_group(_vec(0.0), [near, far], max_distance=0.5)
|
||||
assert chosen is not None and chosen.id == 1
|
||||
|
||||
|
||||
def test_a_candidate_near_nothing_joins_nothing():
|
||||
assert assign_to_group(
|
||||
_vec(0.0), [_group(1, math.pi / 2)], max_distance=0.10,
|
||||
) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_later_variant_joins_the_existing_post_instead_of_founding_one(db):
|
||||
artist, source = await _seed(db, name="rejoin-artist")
|
||||
start = datetime.now(UTC) - timedelta(days=3)
|
||||
await _message(db, source, artist, ext="r1", at=start, vec=_vec(0.0), text="first")
|
||||
await _message(
|
||||
db, source, artist, ext="r2", at=start + timedelta(minutes=2),
|
||||
vec=_vec(0.05), text="second",
|
||||
)
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
|
||||
group = (await db.execute(
|
||||
select(Post).where(Post.synthesized_by == DROP_GROUPER)
|
||||
)).scalar_one()
|
||||
group_id = group.id
|
||||
|
||||
# A day later — far outside the drop window, well inside the open period.
|
||||
await _message(
|
||||
db, source, artist, ext="r3", at=start + timedelta(days=1),
|
||||
vec=_vec(0.06), text="next day",
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
joined = await join_open_groups(
|
||||
db, source, max_distance=0.10, window_minutes=60,
|
||||
close_after_hours=168, resurface_min_images=2,
|
||||
resurface_cooldown_hours=24,
|
||||
)
|
||||
await db.commit()
|
||||
assert joined == 1
|
||||
|
||||
# No second post — that is the point of the step.
|
||||
posts = (await db.execute(
|
||||
select(Post).where(Post.synthesized_by == DROP_GROUPER)
|
||||
)).scalars().all()
|
||||
assert len(posts) == 1
|
||||
|
||||
db.expunge_all()
|
||||
grown = await db.get(Post, group_id)
|
||||
assert grown.last_grew_at is not None
|
||||
assert grown.synthesis_details["message_count"] == 3
|
||||
# The new message's text joined the body, in arrival order.
|
||||
assert grown.description == "first\n\nsecond\n\nnext day"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_variant_after_the_group_closed_starts_a_new_post(db):
|
||||
"""Groups must not stay open forever — artists reuse characters for years."""
|
||||
artist, source = await _seed(db, name="closed-artist")
|
||||
start = datetime.now(UTC) - timedelta(days=60)
|
||||
await _message(db, source, artist, ext="c1", at=start, vec=_vec(0.0))
|
||||
await _message(
|
||||
db, source, artist, ext="c2", at=start + timedelta(minutes=2), vec=_vec(0.05),
|
||||
)
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
|
||||
await _message(
|
||||
db, source, artist, ext="c3", at=datetime.now(UTC) - timedelta(days=1),
|
||||
vec=_vec(0.06),
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
joined = await join_open_groups(
|
||||
db, source, max_distance=0.10, window_minutes=60,
|
||||
close_after_hours=168, resurface_min_images=2,
|
||||
resurface_cooldown_hours=24,
|
||||
)
|
||||
await db.commit()
|
||||
assert joined == 0, "a closed group must not absorb a two-month-later drop"
|
||||
|
||||
created = await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
assert created == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joining_does_not_duplicate_images(db):
|
||||
"""The unique (image, post) constraint is the backstop, not the argument —
|
||||
a second join pass must add nothing at all."""
|
||||
artist, source = await _seed(db, name="nodupe-artist")
|
||||
start = datetime.now(UTC) - timedelta(days=3)
|
||||
await _message(db, source, artist, ext="d1", at=start, vec=_vec(0.0))
|
||||
await _message(
|
||||
db, source, artist, ext="d2", at=start + timedelta(minutes=2), vec=_vec(0.05),
|
||||
)
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
|
||||
await _message(
|
||||
db, source, artist, ext="d3", at=start + timedelta(days=1), vec=_vec(0.06),
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
kwargs = {
|
||||
"max_distance": 0.10, "window_minutes": 60, "close_after_hours": 168,
|
||||
"resurface_min_images": 2, "resurface_cooldown_hours": 24,
|
||||
}
|
||||
assert await join_open_groups(db, source, **kwargs) == 1
|
||||
await db.commit()
|
||||
assert await join_open_groups(db, source, **kwargs) == 0
|
||||
await db.commit()
|
||||
|
||||
group = (await db.execute(
|
||||
select(Post).where(Post.synthesized_by == DROP_GROUPER)
|
||||
)).scalar_one()
|
||||
links = (await db.execute(
|
||||
select(ImageProvenance.image_record_id).where(ImageProvenance.post_id == group.id)
|
||||
)).scalars().all()
|
||||
assert len(links) == len(set(links)) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_group_that_grows_by_one_does_not_move_in_the_feed(db):
|
||||
"""The anti-thrash rule, end to end. A drip-feed updates IN PLACE."""
|
||||
artist, source = await _seed(db, name="dripfeed-artist")
|
||||
start = datetime.now(UTC) - timedelta(days=3)
|
||||
await _message(db, source, artist, ext="t1", at=start, vec=_vec(0.0))
|
||||
await _message(
|
||||
db, source, artist, ext="t2", at=start + timedelta(minutes=2), vec=_vec(0.05),
|
||||
)
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
|
||||
await _message(
|
||||
db, source, artist, ext="t3", at=start + timedelta(days=1), vec=_vec(0.06),
|
||||
)
|
||||
await db.commit()
|
||||
await join_open_groups(
|
||||
db, source, max_distance=0.10, window_minutes=60,
|
||||
close_after_hours=168, resurface_min_images=2, resurface_cooldown_hours=24,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
group = (await db.execute(
|
||||
select(Post).where(Post.synthesized_by == DROP_GROUPER)
|
||||
)).scalar_one()
|
||||
assert group.last_grew_at is not None, "it grew"
|
||||
assert group.resurfaced_at is None, "but one image must not move it"
|
||||
assert group.synthesis_details["images_since_surface"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_second_wave_resurfaces_the_post(db):
|
||||
artist, source = await _seed(db, name="secondwave-artist")
|
||||
start = datetime.now(UTC) - timedelta(days=3)
|
||||
await _message(db, source, artist, ext="w1", at=start, vec=_vec(0.0))
|
||||
await _message(
|
||||
db, source, artist, ext="w2", at=start + timedelta(minutes=2), vec=_vec(0.05),
|
||||
)
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
|
||||
for i, ext in enumerate(("w3", "w4", "w5")):
|
||||
await _message(
|
||||
db, source, artist, ext=ext,
|
||||
at=start + timedelta(days=1, minutes=i), vec=_vec(0.06),
|
||||
)
|
||||
await db.commit()
|
||||
await join_open_groups(
|
||||
db, source, max_distance=0.10, window_minutes=60,
|
||||
close_after_hours=168, resurface_min_images=2, resurface_cooldown_hours=24,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
group = (await db.execute(
|
||||
select(Post).where(Post.synthesized_by == DROP_GROUPER)
|
||||
)).scalar_one()
|
||||
assert group.resurfaced_at is not None
|
||||
# The counter resets, so the NEXT trickle starts from zero rather than
|
||||
# riding the same three images into a second bump.
|
||||
assert group.synthesis_details["images_since_surface"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resurfaced_group_sorts_by_when_it_moved_not_when_it_started(db):
|
||||
"""The feed ordering decision, pinned. The group's drop began three days
|
||||
ago — older than the decoy post — but it resurfaced just now, so it leads."""
|
||||
artist, source = await _seed(db, name="feedorder-artist")
|
||||
start = datetime.now(UTC) - timedelta(days=3)
|
||||
await _message(db, source, artist, ext="f1", at=start, vec=_vec(0.0))
|
||||
await _message(
|
||||
db, source, artist, ext="f2", at=start + timedelta(minutes=2), vec=_vec(0.05),
|
||||
)
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
|
||||
# An ordinary post from yesterday: newer than the drop's start, older than
|
||||
# the moment the group resurfaces.
|
||||
decoy = Post(
|
||||
source_id=source.id, artist_id=artist.id, external_post_id="decoy",
|
||||
post_date=datetime.now(UTC) - timedelta(days=1), post_title="Decoy",
|
||||
)
|
||||
db.add(decoy)
|
||||
await db.flush()
|
||||
decoy_id = decoy.id
|
||||
|
||||
for i, ext in enumerate(("f3", "f4", "f5")):
|
||||
await _message(
|
||||
db, source, artist, ext=ext,
|
||||
at=start + timedelta(days=1, minutes=i), vec=_vec(0.06),
|
||||
)
|
||||
await db.commit()
|
||||
await join_open_groups(
|
||||
db, source, max_distance=0.10, window_minutes=60,
|
||||
close_after_hours=168, resurface_min_images=2, resurface_cooldown_hours=24,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
group = (await db.execute(
|
||||
select(Post).where(Post.synthesized_by == DROP_GROUPER)
|
||||
)).scalar_one()
|
||||
page = await PostFeedService(db).scroll(
|
||||
cursor=None, artist_id=artist.id, limit=50,
|
||||
)
|
||||
ids = [i["id"] for i in page["items"]]
|
||||
assert ids.index(group.id) < ids.index(decoy_id), (
|
||||
"a resurfaced group must lead a post published after the drop STARTED"
|
||||
)
|
||||
# And it is still reported as a grouping that grew.
|
||||
item = next(i for i in page["items"] if i["id"] == group.id)
|
||||
assert item["last_grew_at"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pagination_stays_stable_when_a_group_has_resurfaced(db):
|
||||
"""The cursor is built in Python and the ORDER BY in SQL. If they disagree
|
||||
on resurfaced_at's precedence, rows are silently skipped or repeated at
|
||||
every page boundary — so page through one at a time and check."""
|
||||
artist, source = await _seed(db, name="cursor-artist")
|
||||
start = datetime.now(UTC) - timedelta(days=5)
|
||||
await _message(db, source, artist, ext="x1", at=start, vec=_vec(0.0))
|
||||
await _message(
|
||||
db, source, artist, ext="x2", at=start + timedelta(minutes=2), vec=_vec(0.05),
|
||||
)
|
||||
await db.commit()
|
||||
await group_source(db, source, max_distance=0.10, window_minutes=60)
|
||||
await db.commit()
|
||||
|
||||
for i in range(4):
|
||||
db.add(Post(
|
||||
source_id=source.id, artist_id=artist.id,
|
||||
external_post_id=f"plain{i}",
|
||||
post_date=datetime.now(UTC) - timedelta(days=i + 1),
|
||||
))
|
||||
for i, ext in enumerate(("x3", "x4", "x5")):
|
||||
await _message(
|
||||
db, source, artist, ext=ext,
|
||||
at=start + timedelta(days=1, minutes=i), vec=_vec(0.06),
|
||||
)
|
||||
await db.commit()
|
||||
await join_open_groups(
|
||||
db, source, max_distance=0.10, window_minutes=60,
|
||||
close_after_hours=168, resurface_min_images=2, resurface_cooldown_hours=24,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
svc = PostFeedService(db)
|
||||
everything = [i["id"] for i in (
|
||||
await svc.scroll(cursor=None, artist_id=artist.id, limit=100)
|
||||
)["items"]]
|
||||
|
||||
walked, cursor = [], None
|
||||
while True:
|
||||
page = await svc.scroll(
|
||||
cursor=cursor, artist_id=artist.id, limit=1,
|
||||
)
|
||||
walked.extend(i["id"] for i in page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
assert walked == everything
|
||||
|
||||
Reference in New Issue
Block a user