bvandeusenandClaude Opus 5.5
30a263a47a
The operator's problem: a Patreon teaser is a pointer, and its card showed the censored crop plus a text link while the content it pointed at sat on another card. The fix is a REFERENCE, not an absorption: "the nested items on the unified post are a duplicate or reference of existing content". Nothing is written. Discord posts keep their own rows, dates and places in the feed. - post_unification: for each teaser with a linked association, the drop's images and text, plus its variant family: Discord images sharing the seed's gated LEADING working name, or a phash near-duplicate, within a window of the teaser. One hop only, oldest first. - Measured on artist 8 before writing it: of 121 message pairs 2-60 days apart that share a gated token, 106 share the leading name and all read as real families. Of the 15 sharing only a trailing word, 14 are sibling pieces and one is a plain collision (`bottom`, 56 days). The family cap is 8, not the pairing cap of 6, because `tentacooler` and `0-k1` (6 posts each) are real families. - The feed drops a linked drop's own card only within discord_link_fold_hours of its teaser (default 24): "only hidden from the post view they're posted the same day". Older referenced posts stay where they landed. - post_association.linked_by records whether FC or a person made the link, so the card can say so. Undo is the existing dismiss. - `image0` (gallery-dl's fallback name) becomes a stopword. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
491 lines
22 KiB
Python
491 lines
22 KiB
Python
"""FC-3e: cursor-paginated read service for the Posts stream.
|
|
|
|
Uses the shared `pagination` cursor (base64 of "<iso8601_sort_key>|<id>") so
|
|
every feed paginates identically. Sort key here is COALESCE(Post.post_date,
|
|
Post.downloaded_at) so posts without a publish date sort by when we captured them.
|
|
|
|
Pure read-surface; no writes. The service composes the post dict
|
|
(thumbnails from every image linked to the post — its own primary images
|
|
plus cross-posted duplicates via image_provenance — and non-media
|
|
attachments from PostAttachment) so the API layer can jsonify directly.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from sqlalchemy import and_, func, or_, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from ..models import (
|
|
Artist,
|
|
ExternalLink,
|
|
ImageProvenance,
|
|
ImageRecord,
|
|
ImportSettings,
|
|
Post,
|
|
PostAttachment,
|
|
Source,
|
|
attachment_download_url,
|
|
)
|
|
from ..utils.text import html_to_plain, truncate_at_word
|
|
from .gallery_service import thumbnail_url
|
|
from .pagination import decode_cursor, encode_cursor
|
|
from .post_body import render_post_body
|
|
|
|
DESCRIPTION_LIMIT = 280
|
|
THUMBNAIL_LIMIT = 6
|
|
|
|
|
|
def _sort_key():
|
|
"""Postgres COALESCE expression used in ORDER BY and WHERE clauses.
|
|
|
|
`resurfaced_at` leads (milestone 388 E3). A synthetic post stays OPEN — a
|
|
creator who adds variants the next day extends the existing post — so such
|
|
a post has two dates, and which one orders the feed is a real decision:
|
|
|
|
* ordering by when the drop STARTED buries a group that grows a week later
|
|
under a week of other posts, so the operator never sees the new content —
|
|
which defeats keeping the group open at all;
|
|
* ordering by every growth lets a group that gains one image a day sit
|
|
permanently at the top, so chat out-competes authored posts for the front
|
|
page — the opposite of "post pacing stays front and centre".
|
|
|
|
So the feed orders by neither directly. `resurfaced_at` moves only when the
|
|
anti-thrash rule fires (discord_grouping.should_resurface: enough new
|
|
images AND enough time since the last move), which means a drip-feed
|
|
updates IN PLACE and a genuine second wave resurfaces exactly once.
|
|
|
|
It is NULL on every ordinary post, so this COALESCE cannot move anything
|
|
that is not a grouping. Used identically in ORDER BY and in the cursor's
|
|
WHERE, which is what keeps pagination stable across the change.
|
|
"""
|
|
return func.coalesce(Post.resurfaced_at, Post.post_date, Post.downloaded_at)
|
|
|
|
|
|
def _post_sort_value(post: Post):
|
|
"""The Python twin of `_sort_key()`, for building a cursor from a loaded row.
|
|
|
|
Kept next to it on purpose: these two are one expression in two languages,
|
|
and the failure when they disagree is not an error but a quiet one — rows
|
|
skipped or repeated at page boundaries, which reads as a backend bug
|
|
anywhere but here.
|
|
"""
|
|
return post.resurfaced_at or post.post_date or post.downloaded_at
|
|
|
|
|
|
class PostFeedService:
|
|
def __init__(self, session: AsyncSession):
|
|
self.session = session
|
|
|
|
async def scroll(
|
|
self,
|
|
*,
|
|
cursor: str | None = None,
|
|
artist_id: int | None = None,
|
|
platform: str | None = None,
|
|
q: str | None = None,
|
|
limit: int = 24,
|
|
direction: str = "older",
|
|
) -> dict:
|
|
"""Paginate the feed from `cursor`. direction='older' walks back in
|
|
time (default, infinite-scroll down); direction='newer' walks forward
|
|
(scroll up in an anchored view). Items are always returned in feed
|
|
(descending) order; `next_cursor` points to the far edge in the
|
|
requested direction (null when exhausted).
|
|
|
|
`q` is a free-text filter (ILIKE substring over post_title OR
|
|
description) applied INSIDE the artist/platform scope, so a search
|
|
from the Browse bar stays within whatever artist is filtered in
|
|
view (operator-asked 2026-06-11)."""
|
|
if limit < 1 or limit > 100:
|
|
raise ValueError("limit must be between 1 and 100")
|
|
if direction not in ("older", "newer"):
|
|
raise ValueError("direction must be 'older' or 'newer'")
|
|
|
|
sort_key = _sort_key()
|
|
# Artist via the denormalized Post.artist_id (alembic 0030);
|
|
# Source via LEFT JOIN since post.source_id can now be NULL for
|
|
# filesystem-imported posts with no live subscription. A
|
|
# platform= filter implicitly excludes NULL-source posts (they
|
|
# have no platform); an artist_id= filter still surfaces them
|
|
# because Post.artist_id is always set.
|
|
stmt = (
|
|
select(Post, Artist, Source)
|
|
.join(Artist, Post.artist_id == Artist.id)
|
|
.outerjoin(Source, Post.source_id == Source.id)
|
|
)
|
|
# Absorbed posts are the individual chat messages a synthetic post
|
|
# replaced (milestone 388 E2). They stay in the table — they are the
|
|
# images' true origin and the grouping has to be auditable — but the
|
|
# feed shows the post FC authored, not the dozen lines it was built
|
|
# from. `around` and `get_post` deliberately do NOT apply this: reaching
|
|
# a member by id is how you inspect a grouping.
|
|
stmt = stmt.where(Post.absorbed_by_post_id.is_(None))
|
|
# A linked Discord drop is shown ON its teaser's card by reference
|
|
# (#4402), so its own card sitting beside that teaser is the same
|
|
# release twice. Only then is it left out — an older drop keeps its
|
|
# place, because a reference does not take anything out of history.
|
|
fold_hours = await self._fold_hours()
|
|
if fold_hours > 0:
|
|
from .post_unification import fold_clause
|
|
|
|
stmt = stmt.where(fold_clause(fold_hours))
|
|
if artist_id is not None:
|
|
stmt = stmt.where(Post.artist_id == artist_id)
|
|
if platform is not None:
|
|
stmt = stmt.where(Source.platform == platform)
|
|
if q:
|
|
like = f"%{q}%"
|
|
stmt = stmt.where(or_(
|
|
Post.post_title.ilike(like),
|
|
Post.description.ilike(like),
|
|
))
|
|
if cursor:
|
|
cur_ts, cur_id = decode_cursor(cursor)
|
|
if direction == "older":
|
|
stmt = stmt.where(or_(
|
|
sort_key < cur_ts,
|
|
and_(sort_key == cur_ts, Post.id < cur_id),
|
|
))
|
|
else:
|
|
stmt = stmt.where(or_(
|
|
sort_key > cur_ts,
|
|
and_(sort_key == cur_ts, Post.id > cur_id),
|
|
))
|
|
|
|
if direction == "older":
|
|
stmt = stmt.order_by(sort_key.desc(), Post.id.desc())
|
|
else:
|
|
stmt = stmt.order_by(sort_key.asc(), Post.id.asc())
|
|
stmt = stmt.limit(limit + 1)
|
|
rows = (await self.session.execute(stmt)).all()
|
|
|
|
has_more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
if direction == "newer":
|
|
# Fetched ascending (closest-newer first); flip to feed order.
|
|
rows = list(reversed(rows))
|
|
|
|
next_cursor: str | None = None
|
|
if has_more and rows:
|
|
# Far edge in the travel direction: oldest row going older,
|
|
# newest row going newer (rows is descending for display).
|
|
edge_post = rows[-1][0] if direction == "older" else rows[0][0]
|
|
# Must match _sort_key() exactly, including resurfaced_at's
|
|
# precedence: a cursor built from a different expression than the
|
|
# ORDER BY silently skips or repeats rows at every page boundary.
|
|
edge_key = _post_sort_value(edge_post)
|
|
next_cursor = encode_cursor(edge_key, edge_post.id)
|
|
|
|
post_ids = [p.id for p, _, _ in rows]
|
|
thumbs_map = await self._thumbnails_for(post_ids)
|
|
atts_map = await self._attachments_for(post_ids)
|
|
links_map = await self._links_for(post_ids)
|
|
unified_map = await self._unified_for([p for p, _, _ in rows])
|
|
|
|
items = [
|
|
self._to_dict(
|
|
post, artist, source, thumbs_map, atts_map, links_map, unified_map,
|
|
)
|
|
for post, artist, source in rows
|
|
]
|
|
return {"items": items, "next_cursor": next_cursor}
|
|
|
|
async def around(
|
|
self,
|
|
*,
|
|
post_id: int,
|
|
artist_id: int | None = None,
|
|
platform: str | None = None,
|
|
q: str | None = None,
|
|
limit: int = 12,
|
|
) -> dict | None:
|
|
"""A window centered on `post_id`: up to `limit` newer posts + the
|
|
post + up to `limit` older posts, in feed (descending) order, with a
|
|
cursor for each end. Returns None if the post doesn't exist."""
|
|
anchor = (await self.session.execute(
|
|
select(Post, Artist, Source)
|
|
.join(Artist, Post.artist_id == Artist.id)
|
|
.outerjoin(Source, Post.source_id == Source.id)
|
|
.where(Post.id == post_id)
|
|
)).one_or_none()
|
|
if anchor is None:
|
|
return None
|
|
anchor_post, anchor_artist, anchor_source = anchor
|
|
anchor_key = _post_sort_value(anchor_post)
|
|
anchor_cursor = encode_cursor(anchor_key, anchor_post.id)
|
|
|
|
older = await self.scroll(
|
|
cursor=anchor_cursor, artist_id=artist_id, platform=platform,
|
|
q=q, limit=limit, direction="older",
|
|
)
|
|
newer = await self.scroll(
|
|
cursor=anchor_cursor, artist_id=artist_id, platform=platform,
|
|
q=q, limit=limit, direction="newer",
|
|
)
|
|
thumbs_map = await self._thumbnails_for([anchor_post.id])
|
|
atts_map = await self._attachments_for([anchor_post.id])
|
|
anchor_item = self._to_dict(
|
|
anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map,
|
|
await self._links_for([anchor_post.id]),
|
|
await self._unified_for([anchor_post]),
|
|
)
|
|
return {
|
|
"items": newer["items"] + [anchor_item] + older["items"],
|
|
"cursor_older": older["next_cursor"],
|
|
"cursor_newer": newer["next_cursor"],
|
|
"anchor_id": anchor_post.id,
|
|
}
|
|
|
|
async def get_post(self, post_id: int) -> dict | None:
|
|
row = (await self.session.execute(
|
|
select(Post, Artist, Source)
|
|
.join(Artist, Post.artist_id == Artist.id)
|
|
.outerjoin(Source, Post.source_id == Source.id)
|
|
.where(Post.id == post_id)
|
|
)).one_or_none()
|
|
if row is None:
|
|
return None
|
|
post, artist, source = row
|
|
# Detail endpoint returns the FULL image list for PostModal's
|
|
# masonry grid — feed query still caps at THUMBNAIL_LIMIT via
|
|
# the default arg.
|
|
thumbs_map = await self._thumbnails_for([post.id], limit=None)
|
|
atts_map = await self._attachments_for([post.id])
|
|
item = self._to_dict(
|
|
post, artist, source, thumbs_map, atts_map,
|
|
await self._links_for([post.id]),
|
|
await self._unified_for([post]),
|
|
)
|
|
item["description_full"] = html_to_plain(post.description)
|
|
# Full (uncapped) translated description for the detail view (#143).
|
|
item["description_translated_full"] = post.description_translated
|
|
# Rendered body for faithful (semantic) display in the post view;
|
|
# detail-only, which is what keeps the feed list lightweight plain text
|
|
# (measured: note #3962). None when the post has no body. What
|
|
# "rendered" involves — sanitize, then repoint inline images at our own
|
|
# copies — belongs to `render_post_body`, which the provenance panel
|
|
# shares so the two surfaces cannot drift apart again (#3965).
|
|
item["description_html"] = await render_post_body(
|
|
self.session, post.description, post.artist_id,
|
|
)
|
|
item["external_links"] = await self._external_links_for(post.id)
|
|
return item
|
|
|
|
async def _external_links_for(self, post_id: int) -> list[dict]:
|
|
"""Off-platform file-host links recorded for a post (detail-only). Each
|
|
carries its host, full url, label, and download status so the post view
|
|
can surface them (and, later, a retry/download affordance)."""
|
|
rows = (await self.session.execute(
|
|
select(ExternalLink)
|
|
.where(ExternalLink.post_id == post_id)
|
|
.order_by(ExternalLink.id.asc())
|
|
)).scalars().all()
|
|
return [
|
|
{
|
|
"id": e.id, "host": e.host, "url": e.url,
|
|
"label": e.label, "status": e.status,
|
|
}
|
|
for e in rows
|
|
]
|
|
|
|
# --- composition helpers ---------------------------------------------
|
|
|
|
async def _thumbnails_for(
|
|
self, post_ids: list[int], *, limit: int | None = THUMBNAIL_LIMIT,
|
|
) -> dict[int, dict]:
|
|
"""post_id -> {"thumbs": [...up to limit], "more": int}.
|
|
|
|
Selects up to `limit` images per post via window function so we
|
|
can detect overflow in a single query. Pass `limit=None` to
|
|
return ALL thumbnails per post (used by `get_post` for PostModal's
|
|
masonry grid; the feed pass keeps the default cap so payloads
|
|
stay small).
|
|
"""
|
|
if not post_ids:
|
|
return {}
|
|
# A post shows EVERY image linked to it — both its own primary images and
|
|
# cross-posted duplicates linked via image_provenance (a near-dup of an
|
|
# existing image gets a provenance row for the new post, not dropped; see
|
|
# image_provenance docstring + the importer enrich-on-duplicate path). The
|
|
# UNION dedups (image, post) pairs and also keeps any legacy image that has
|
|
# a primary_post_id but no provenance row. Partition the window on the
|
|
# link's post_id, not ImageRecord.primary_post_id, so a duplicate counts
|
|
# under each post it belongs to.
|
|
links = (
|
|
select(
|
|
ImageProvenance.image_record_id.label("image_id"),
|
|
ImageProvenance.post_id.label("post_id"),
|
|
)
|
|
.where(ImageProvenance.post_id.in_(post_ids))
|
|
.union(
|
|
select(
|
|
ImageRecord.id.label("image_id"),
|
|
ImageRecord.primary_post_id.label("post_id"),
|
|
).where(ImageRecord.primary_post_id.in_(post_ids))
|
|
)
|
|
.subquery()
|
|
)
|
|
# Rank images within each post; cap at `limit` rows per post when
|
|
# limit is set, return all when limit is None.
|
|
ranked = (
|
|
select(
|
|
ImageRecord.id,
|
|
links.c.post_id,
|
|
ImageRecord.sha256,
|
|
ImageRecord.mime,
|
|
ImageRecord.thumbnail_path,
|
|
func.row_number().over(
|
|
partition_by=links.c.post_id,
|
|
order_by=ImageRecord.id.asc(),
|
|
).label("rn"),
|
|
func.count(ImageRecord.id).over(
|
|
partition_by=links.c.post_id,
|
|
).label("total"),
|
|
)
|
|
.join(ImageRecord, ImageRecord.id == links.c.image_id)
|
|
.subquery()
|
|
)
|
|
stmt = select(
|
|
ranked.c.id, ranked.c.post_id,
|
|
ranked.c.sha256, ranked.c.mime, ranked.c.thumbnail_path, ranked.c.total,
|
|
)
|
|
if limit is not None:
|
|
stmt = stmt.where(ranked.c.rn <= limit)
|
|
rows = (await self.session.execute(stmt)).all()
|
|
|
|
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
|
|
for img_id, pid, sha, mime, tp, total in rows:
|
|
entry = out.setdefault(pid, {"thumbs": [], "more": 0})
|
|
entry["thumbs"].append({
|
|
"image_id": img_id,
|
|
"thumbnail_url": thumbnail_url(tp, sha, mime),
|
|
"mime": mime,
|
|
})
|
|
# `total` is constant per partition; overflow = total - THUMBNAIL_LIMIT.
|
|
entry["more"] = max(0, total - THUMBNAIL_LIMIT)
|
|
return out
|
|
|
|
async def _attachments_for(self, post_ids: list[int]) -> dict[int, list[dict]]:
|
|
if not post_ids:
|
|
return {}
|
|
rows = (await self.session.execute(
|
|
select(PostAttachment)
|
|
.where(PostAttachment.post_id.in_(post_ids))
|
|
.order_by(PostAttachment.id.asc())
|
|
)).scalars().all()
|
|
out: dict[int, list[dict]] = {pid: [] for pid in post_ids}
|
|
for att in rows:
|
|
out.setdefault(att.post_id, []).append({
|
|
"id": att.id,
|
|
"original_filename": att.original_filename,
|
|
"ext": att.ext,
|
|
"mime": att.mime,
|
|
"size_bytes": att.size_bytes,
|
|
"download_url": attachment_download_url(att.id),
|
|
})
|
|
return out
|
|
|
|
async def _links_for(self, post_ids: list[int]) -> dict[int, list[dict]]:
|
|
"""Accepted announcement links touching these posts (#388 E5).
|
|
|
|
Only ACCEPTED ones. A pending proposal is a question for the review
|
|
queue, not a claim to render beside the artwork — showing one here
|
|
would assert a link the operator has not agreed to, which is the exact
|
|
failure the confirm-only design exists to prevent.
|
|
"""
|
|
# Imported here rather than at module scope: post_association_service
|
|
# imports discord_grouping, which imports the models, and the feed
|
|
# service is imported by the API at startup. A local import keeps that
|
|
# chain out of the import graph for a purely optional read.
|
|
from .post_association_service import PostAssociationService
|
|
|
|
return await PostAssociationService(self.session).linked_for(post_ids)
|
|
|
|
async def _unified_for(self, posts: list[Post]) -> dict[int, dict]:
|
|
"""The reference set each teaser's card shows (#4402). Local import for
|
|
the same reason as `_links_for`: it reaches the association service."""
|
|
from .post_unification import PostUnificationService
|
|
|
|
return await PostUnificationService(self.session).unified_for(posts)
|
|
|
|
async def _fold_hours(self) -> float:
|
|
"""`discord_link_fold_hours`, read without assuming the row exists.
|
|
|
|
The feed is the one surface that must not fail on a settings row the
|
|
caller never needed, so a missing row folds nothing rather than
|
|
raising — which is also exactly how the feed behaved before this.
|
|
"""
|
|
settings = await self.session.get(ImportSettings, 1)
|
|
return float(settings.discord_link_fold_hours) if settings is not None else 0.0
|
|
|
|
def _to_dict(
|
|
self, post: Post, artist: Artist, source: Source | None,
|
|
thumbs_map: dict, atts_map: dict, links_map: dict | None = None,
|
|
unified_map: dict | None = None,
|
|
) -> dict:
|
|
plain_full = html_to_plain(post.description) if post.description else None
|
|
if plain_full is None:
|
|
description_plain, truncated = None, False
|
|
else:
|
|
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
|
# Translation (#143): the stored translated description is already plain
|
|
# text; truncate it the same way for the card.
|
|
desc_trans = post.description_translated
|
|
desc_trans_short = (
|
|
truncate_at_word(desc_trans, DESCRIPTION_LIMIT)[0] if desc_trans else None
|
|
)
|
|
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
|
# `source` is null for filesystem-imported posts with no live
|
|
# subscription (alembic 0030). Frontend renders that as a
|
|
# "filesystem import" affordance instead of a platform chip.
|
|
return {
|
|
"id": post.id,
|
|
"external_post_id": post.external_post_id,
|
|
"post_url": post.post_url,
|
|
"post_title": post.post_title,
|
|
"post_date": post.post_date.isoformat() if post.post_date else None,
|
|
"downloaded_at": post.downloaded_at.isoformat(),
|
|
"description_plain": description_plain,
|
|
"description_truncated": truncated,
|
|
# Translation-forward fields (#143): shown by default when present;
|
|
# UI toggles back to the originals above. Source lang labels the toggle.
|
|
"post_title_translated": post.post_title_translated,
|
|
"description_translated": desc_trans_short,
|
|
"translated_source_lang": post.translated_source_lang,
|
|
# Sticky per-post translation choice (auto/force/original, #155).
|
|
"translation_override": post.translation_override,
|
|
# Milestone 388 E2. Non-null means FC AUTHORED this post by grouping
|
|
# a creator's drop — the UI must say so wherever the post appears,
|
|
# and `synthesis` carries what it was built from so the operator can
|
|
# audit a grouping FC invented. Null for every real post; the two
|
|
# keys are always present so the frontend never branches on absence.
|
|
"synthesized_by": post.synthesized_by,
|
|
"synthesis": post.synthesis_details,
|
|
# #388 E3. A grouping stays open, so the card can say "updated N
|
|
# ago" — which is the whole signal that chat content is trickling
|
|
# in. NULL means it has not grown since it was created.
|
|
"last_grew_at": post.last_grew_at.isoformat() if post.last_grew_at else None,
|
|
# Accepted links only (#388 E5): [{role, post_id, id}], where role
|
|
# is "announces" (this post is the teaser) or "announced_by" (this
|
|
# post is the drop). Always a list so the UI never branches on
|
|
# absence.
|
|
"associations": (links_map or {}).get(post.id, []),
|
|
# #4402. On a teaser with a linked drop: what the card shows BY
|
|
# REFERENCE — the drop's images, the piece's older variants, and
|
|
# the text of each — plus who made each link, so one FC made by
|
|
# itself can say so and offer the undo. None on every other post.
|
|
"unified": (unified_map or {}).get(post.id),
|
|
# Non-null on a chat message a synthetic post absorbed. The feed
|
|
# filters these out, but `around`/`get_post` still reach them, and
|
|
# the UI uses this to explain why a post it linked to is not in the
|
|
# stream.
|
|
"absorbed_by_post_id": post.absorbed_by_post_id,
|
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
|
"source": (
|
|
{"id": source.id, "platform": source.platform}
|
|
if source is not None else None
|
|
),
|
|
"thumbnails": thumbs_entry["thumbs"],
|
|
"thumbnails_more": thumbs_entry["more"],
|
|
"attachments": atts_map.get(post.id, []),
|
|
}
|