bvandeusen
1ac448d881
Items 2-5 of #3072. Item 1 (the per-row sweep inserts) is separate. 2. .fc-bad was not merely duplicated — it is .fc-weak under a second name. Both local definitions were `color: rgb(var(--v-theme-error))`, identical to the global .fc-weak, and GpuAgentCard was already using .fc-weak to colour exactly what GpuActivityPanel coloured .fc-bad (an errored count, red when non-zero). So rather than promoting a synonym to app.css, both call sites now use .fc-weak and the local defs are gone. app.css's status-colour comment records why there is no .fc-bad, next to the existing note on why .fc-ok is deliberately NOT global. 3. GalleryItem.vue's obsidian literals now use --v-theme-background, which IS obsidian (vuetify-theme.js maps background -> surfaces. obsidian). Preferred over --fc-chrome-rgb: same value, but that variable is named for the nav fade, not for the palette entry. The ticket said these were the only three real uses in the tree. They are not — GalleryItem itself had two more in the artist-label gradient (fixed here, so the file is now consistent), and ~13 more live in SeriesView, SeriesReaderView, ImageViewer, ArtistHeader, ExploreView and GalleryFilterBar. Those are a separate sweep, filed rather than folded in here. 4. The attachment download path had two hand-formatted copies. One definition now, `attachment_download_url`, next to the model both serializers already import. The test pins it by MATCHING the built path against the app's real URL map rather than comparing to a literal — a string-equality test would still pass after someone renamed the route, which is the drift the helper exists to prevent. 5. Extension API key now compares with hmac.compare_digest. Compared as BYTES, not str: compare_digest's str form raises TypeError on non-ASCII, and this value comes straight from an attacker-controlled header, so the str form would turn a junk key into a 500 instead of a 403. Low stakes either way — the API is unauthenticated-by-design on a LAN — but it costs nothing. Refs #3072
412 lines
17 KiB
Python
412 lines
17 KiB
Python
"""FC-3e: cursor-paginated read service for the Posts stream.
|
|
|
|
Uses the shared `pagination` cursor (base64 of "<iso8601_sort_key>|<id>") so
|
|
every feed paginates identically. Sort key here is COALESCE(Post.post_date,
|
|
Post.downloaded_at) so posts without a publish date sort by when we captured them.
|
|
|
|
Pure read-surface; no writes. The service composes the post dict
|
|
(thumbnails from every image linked to the post — its own primary images
|
|
plus cross-posted duplicates via image_provenance — and non-media
|
|
attachments from PostAttachment) so the API layer can jsonify directly.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from html import unescape
|
|
|
|
from sqlalchemy import and_, func, or_, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from ..models import (
|
|
Artist,
|
|
ExternalLink,
|
|
ImageProvenance,
|
|
ImageRecord,
|
|
Post,
|
|
PostAttachment,
|
|
Source,
|
|
attachment_download_url,
|
|
)
|
|
from ..utils.html_sanitize import (
|
|
extract_img_srcs,
|
|
rewrite_img_srcs,
|
|
sanitize_post_html,
|
|
)
|
|
from ..utils.paths import filehash_from_url
|
|
from ..utils.text import html_to_plain, truncate_at_word
|
|
from .gallery_service import image_url, thumbnail_url
|
|
from .pagination import decode_cursor, encode_cursor
|
|
|
|
DESCRIPTION_LIMIT = 280
|
|
THUMBNAIL_LIMIT = 6
|
|
|
|
|
|
def _sort_key():
|
|
"""Postgres COALESCE expression used in ORDER BY and WHERE clauses."""
|
|
return func.coalesce(Post.post_date, Post.downloaded_at)
|
|
|
|
|
|
class PostFeedService:
|
|
def __init__(self, session: AsyncSession):
|
|
self.session = session
|
|
|
|
async def scroll(
|
|
self,
|
|
*,
|
|
cursor: str | None = None,
|
|
artist_id: int | None = None,
|
|
platform: str | None = None,
|
|
q: str | None = None,
|
|
limit: int = 24,
|
|
direction: str = "older",
|
|
) -> dict:
|
|
"""Paginate the feed from `cursor`. direction='older' walks back in
|
|
time (default, infinite-scroll down); direction='newer' walks forward
|
|
(scroll up in an anchored view). Items are always returned in feed
|
|
(descending) order; `next_cursor` points to the far edge in the
|
|
requested direction (null when exhausted).
|
|
|
|
`q` is a free-text filter (ILIKE substring over post_title OR
|
|
description) applied INSIDE the artist/platform scope, so a search
|
|
from the Browse bar stays within whatever artist is filtered in
|
|
view (operator-asked 2026-06-11)."""
|
|
if limit < 1 or limit > 100:
|
|
raise ValueError("limit must be between 1 and 100")
|
|
if direction not in ("older", "newer"):
|
|
raise ValueError("direction must be 'older' or 'newer'")
|
|
|
|
sort_key = _sort_key()
|
|
# Artist via the denormalized Post.artist_id (alembic 0030);
|
|
# Source via LEFT JOIN since post.source_id can now be NULL for
|
|
# filesystem-imported posts with no live subscription. A
|
|
# platform= filter implicitly excludes NULL-source posts (they
|
|
# have no platform); an artist_id= filter still surfaces them
|
|
# because Post.artist_id is always set.
|
|
stmt = (
|
|
select(Post, Artist, Source)
|
|
.join(Artist, Post.artist_id == Artist.id)
|
|
.outerjoin(Source, Post.source_id == Source.id)
|
|
)
|
|
if artist_id is not None:
|
|
stmt = stmt.where(Post.artist_id == artist_id)
|
|
if platform is not None:
|
|
stmt = stmt.where(Source.platform == platform)
|
|
if q:
|
|
like = f"%{q}%"
|
|
stmt = stmt.where(or_(
|
|
Post.post_title.ilike(like),
|
|
Post.description.ilike(like),
|
|
))
|
|
if cursor:
|
|
cur_ts, cur_id = decode_cursor(cursor)
|
|
if direction == "older":
|
|
stmt = stmt.where(or_(
|
|
sort_key < cur_ts,
|
|
and_(sort_key == cur_ts, Post.id < cur_id),
|
|
))
|
|
else:
|
|
stmt = stmt.where(or_(
|
|
sort_key > cur_ts,
|
|
and_(sort_key == cur_ts, Post.id > cur_id),
|
|
))
|
|
|
|
if direction == "older":
|
|
stmt = stmt.order_by(sort_key.desc(), Post.id.desc())
|
|
else:
|
|
stmt = stmt.order_by(sort_key.asc(), Post.id.asc())
|
|
stmt = stmt.limit(limit + 1)
|
|
rows = (await self.session.execute(stmt)).all()
|
|
|
|
has_more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
if direction == "newer":
|
|
# Fetched ascending (closest-newer first); flip to feed order.
|
|
rows = list(reversed(rows))
|
|
|
|
next_cursor: str | None = None
|
|
if has_more and rows:
|
|
# Far edge in the travel direction: oldest row going older,
|
|
# newest row going newer (rows is descending for display).
|
|
edge_post = rows[-1][0] if direction == "older" else rows[0][0]
|
|
edge_key = edge_post.post_date or edge_post.downloaded_at
|
|
next_cursor = encode_cursor(edge_key, edge_post.id)
|
|
|
|
post_ids = [p.id for p, _, _ in rows]
|
|
thumbs_map = await self._thumbnails_for(post_ids)
|
|
atts_map = await self._attachments_for(post_ids)
|
|
|
|
items = [
|
|
self._to_dict(post, artist, source, thumbs_map, atts_map)
|
|
for post, artist, source in rows
|
|
]
|
|
return {"items": items, "next_cursor": next_cursor}
|
|
|
|
async def around(
|
|
self,
|
|
*,
|
|
post_id: int,
|
|
artist_id: int | None = None,
|
|
platform: str | None = None,
|
|
q: str | None = None,
|
|
limit: int = 12,
|
|
) -> dict | None:
|
|
"""A window centered on `post_id`: up to `limit` newer posts + the
|
|
post + up to `limit` older posts, in feed (descending) order, with a
|
|
cursor for each end. Returns None if the post doesn't exist."""
|
|
anchor = (await self.session.execute(
|
|
select(Post, Artist, Source)
|
|
.join(Artist, Post.artist_id == Artist.id)
|
|
.outerjoin(Source, Post.source_id == Source.id)
|
|
.where(Post.id == post_id)
|
|
)).one_or_none()
|
|
if anchor is None:
|
|
return None
|
|
anchor_post, anchor_artist, anchor_source = anchor
|
|
anchor_key = anchor_post.post_date or anchor_post.downloaded_at
|
|
anchor_cursor = encode_cursor(anchor_key, anchor_post.id)
|
|
|
|
older = await self.scroll(
|
|
cursor=anchor_cursor, artist_id=artist_id, platform=platform,
|
|
q=q, limit=limit, direction="older",
|
|
)
|
|
newer = await self.scroll(
|
|
cursor=anchor_cursor, artist_id=artist_id, platform=platform,
|
|
q=q, limit=limit, direction="newer",
|
|
)
|
|
thumbs_map = await self._thumbnails_for([anchor_post.id])
|
|
atts_map = await self._attachments_for([anchor_post.id])
|
|
anchor_item = self._to_dict(
|
|
anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map,
|
|
)
|
|
return {
|
|
"items": newer["items"] + [anchor_item] + older["items"],
|
|
"cursor_older": older["next_cursor"],
|
|
"cursor_newer": newer["next_cursor"],
|
|
"anchor_id": anchor_post.id,
|
|
}
|
|
|
|
async def get_post(self, post_id: int) -> dict | None:
|
|
row = (await self.session.execute(
|
|
select(Post, Artist, Source)
|
|
.join(Artist, Post.artist_id == Artist.id)
|
|
.outerjoin(Source, Post.source_id == Source.id)
|
|
.where(Post.id == post_id)
|
|
)).one_or_none()
|
|
if row is None:
|
|
return None
|
|
post, artist, source = row
|
|
# Detail endpoint returns the FULL image list for PostModal's
|
|
# masonry grid — feed query still caps at THUMBNAIL_LIMIT via
|
|
# the default arg.
|
|
thumbs_map = await self._thumbnails_for([post.id], limit=None)
|
|
atts_map = await self._attachments_for([post.id])
|
|
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
|
item["description_full"] = html_to_plain(post.description)
|
|
# Full (uncapped) translated description for the detail view (#143).
|
|
item["description_translated_full"] = post.description_translated
|
|
# Sanitized HTML body for faithful (semantic) rendering in the post view;
|
|
# detail-only (the feed list stays lightweight plain text). None when the
|
|
# post has no body. Inline `<img>` sources are remapped to locally-served
|
|
# copies (#830 Phase 2) so the body never hotlinks the public CDN.
|
|
item["description_html"] = await self._localize_inline_images(
|
|
sanitize_post_html(post.description), post.artist_id,
|
|
)
|
|
item["external_links"] = await self._external_links_for(post.id)
|
|
return item
|
|
|
|
async def _localize_inline_images(
|
|
self, html: str | None, artist_id: int | None,
|
|
) -> str | None:
|
|
"""Rewrite a post body's inline `<img src=CDN>` to locally-served copies.
|
|
|
|
The join key is the CDN filehash the downloader persisted on each
|
|
ImageRecord (source_filehash): for every body image whose filehash maps
|
|
to a stored image of THIS artist, swap the src to /images/<path>. Images
|
|
we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
|
|
they keep hotlinking, which is the prior behavior. Scoped to the post's
|
|
artist so one creator's body never resolves to another's file."""
|
|
if not html or artist_id is None:
|
|
return html
|
|
srcs = extract_img_srcs(html)
|
|
if not srcs:
|
|
return html
|
|
# filehash -> the raw (as-in-HTML) src strings carrying it. A body can
|
|
# repeat the same image; keep every raw form so each is substituted.
|
|
by_hash: dict[str, list[str]] = {}
|
|
for raw in srcs:
|
|
fh = filehash_from_url(unescape(raw))
|
|
if fh:
|
|
by_hash.setdefault(fh, []).append(raw)
|
|
if not by_hash:
|
|
return html
|
|
rows = (await self.session.execute(
|
|
select(ImageRecord.source_filehash, ImageRecord.path)
|
|
.where(
|
|
ImageRecord.artist_id == artist_id,
|
|
ImageRecord.source_filehash.in_(list(by_hash)),
|
|
)
|
|
)).all()
|
|
replace: dict[str, str] = {}
|
|
for fh, path in rows:
|
|
for raw in by_hash.get(fh, ()):
|
|
replace[raw] = image_url(path)
|
|
return rewrite_img_srcs(html, replace)
|
|
|
|
async def _external_links_for(self, post_id: int) -> list[dict]:
|
|
"""Off-platform file-host links recorded for a post (detail-only). Each
|
|
carries its host, full url, label, and download status so the post view
|
|
can surface them (and, later, a retry/download affordance)."""
|
|
rows = (await self.session.execute(
|
|
select(ExternalLink)
|
|
.where(ExternalLink.post_id == post_id)
|
|
.order_by(ExternalLink.id.asc())
|
|
)).scalars().all()
|
|
return [
|
|
{
|
|
"id": e.id, "host": e.host, "url": e.url,
|
|
"label": e.label, "status": e.status,
|
|
}
|
|
for e in rows
|
|
]
|
|
|
|
# --- composition helpers ---------------------------------------------
|
|
|
|
async def _thumbnails_for(
|
|
self, post_ids: list[int], *, limit: int | None = THUMBNAIL_LIMIT,
|
|
) -> dict[int, dict]:
|
|
"""post_id -> {"thumbs": [...up to limit], "more": int}.
|
|
|
|
Selects up to `limit` images per post via window function so we
|
|
can detect overflow in a single query. Pass `limit=None` to
|
|
return ALL thumbnails per post (used by `get_post` for PostModal's
|
|
masonry grid; the feed pass keeps the default cap so payloads
|
|
stay small).
|
|
"""
|
|
if not post_ids:
|
|
return {}
|
|
# A post shows EVERY image linked to it — both its own primary images and
|
|
# cross-posted duplicates linked via image_provenance (a near-dup of an
|
|
# existing image gets a provenance row for the new post, not dropped; see
|
|
# image_provenance docstring + the importer enrich-on-duplicate path). The
|
|
# UNION dedups (image, post) pairs and also keeps any legacy image that has
|
|
# a primary_post_id but no provenance row. Partition the window on the
|
|
# link's post_id, not ImageRecord.primary_post_id, so a duplicate counts
|
|
# under each post it belongs to.
|
|
links = (
|
|
select(
|
|
ImageProvenance.image_record_id.label("image_id"),
|
|
ImageProvenance.post_id.label("post_id"),
|
|
)
|
|
.where(ImageProvenance.post_id.in_(post_ids))
|
|
.union(
|
|
select(
|
|
ImageRecord.id.label("image_id"),
|
|
ImageRecord.primary_post_id.label("post_id"),
|
|
).where(ImageRecord.primary_post_id.in_(post_ids))
|
|
)
|
|
.subquery()
|
|
)
|
|
# Rank images within each post; cap at `limit` rows per post when
|
|
# limit is set, return all when limit is None.
|
|
ranked = (
|
|
select(
|
|
ImageRecord.id,
|
|
links.c.post_id,
|
|
ImageRecord.sha256,
|
|
ImageRecord.mime,
|
|
ImageRecord.thumbnail_path,
|
|
func.row_number().over(
|
|
partition_by=links.c.post_id,
|
|
order_by=ImageRecord.id.asc(),
|
|
).label("rn"),
|
|
func.count(ImageRecord.id).over(
|
|
partition_by=links.c.post_id,
|
|
).label("total"),
|
|
)
|
|
.join(ImageRecord, ImageRecord.id == links.c.image_id)
|
|
.subquery()
|
|
)
|
|
stmt = select(
|
|
ranked.c.id, ranked.c.post_id,
|
|
ranked.c.sha256, ranked.c.mime, ranked.c.thumbnail_path, ranked.c.total,
|
|
)
|
|
if limit is not None:
|
|
stmt = stmt.where(ranked.c.rn <= limit)
|
|
rows = (await self.session.execute(stmt)).all()
|
|
|
|
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
|
|
for img_id, pid, sha, mime, tp, total in rows:
|
|
entry = out.setdefault(pid, {"thumbs": [], "more": 0})
|
|
entry["thumbs"].append({
|
|
"image_id": img_id,
|
|
"thumbnail_url": thumbnail_url(tp, sha, mime),
|
|
"mime": mime,
|
|
})
|
|
# `total` is constant per partition; overflow = total - THUMBNAIL_LIMIT.
|
|
entry["more"] = max(0, total - THUMBNAIL_LIMIT)
|
|
return out
|
|
|
|
async def _attachments_for(self, post_ids: list[int]) -> dict[int, list[dict]]:
|
|
if not post_ids:
|
|
return {}
|
|
rows = (await self.session.execute(
|
|
select(PostAttachment)
|
|
.where(PostAttachment.post_id.in_(post_ids))
|
|
.order_by(PostAttachment.id.asc())
|
|
)).scalars().all()
|
|
out: dict[int, list[dict]] = {pid: [] for pid in post_ids}
|
|
for att in rows:
|
|
out.setdefault(att.post_id, []).append({
|
|
"id": att.id,
|
|
"original_filename": att.original_filename,
|
|
"ext": att.ext,
|
|
"mime": att.mime,
|
|
"size_bytes": att.size_bytes,
|
|
"download_url": attachment_download_url(att.id),
|
|
})
|
|
return out
|
|
|
|
def _to_dict(
|
|
self, post: Post, artist: Artist, source: Source | None,
|
|
thumbs_map: dict, atts_map: dict,
|
|
) -> dict:
|
|
plain_full = html_to_plain(post.description) if post.description else None
|
|
if plain_full is None:
|
|
description_plain, truncated = None, False
|
|
else:
|
|
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
|
# Translation (#143): the stored translated description is already plain
|
|
# text; truncate it the same way for the card.
|
|
desc_trans = post.description_translated
|
|
desc_trans_short = (
|
|
truncate_at_word(desc_trans, DESCRIPTION_LIMIT)[0] if desc_trans else None
|
|
)
|
|
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
|
# `source` is null for filesystem-imported posts with no live
|
|
# subscription (alembic 0030). Frontend renders that as a
|
|
# "filesystem import" affordance instead of a platform chip.
|
|
return {
|
|
"id": post.id,
|
|
"external_post_id": post.external_post_id,
|
|
"post_url": post.post_url,
|
|
"post_title": post.post_title,
|
|
"post_date": post.post_date.isoformat() if post.post_date else None,
|
|
"downloaded_at": post.downloaded_at.isoformat(),
|
|
"description_plain": description_plain,
|
|
"description_truncated": truncated,
|
|
# Translation-forward fields (#143): shown by default when present;
|
|
# UI toggles back to the originals above. Source lang labels the toggle.
|
|
"post_title_translated": post.post_title_translated,
|
|
"description_translated": desc_trans_short,
|
|
"translated_source_lang": post.translated_source_lang,
|
|
# Sticky per-post translation choice (auto/force/original, #155).
|
|
"translation_override": post.translation_override,
|
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
|
"source": (
|
|
{"id": source.id, "platform": source.platform}
|
|
if source is not None else None
|
|
),
|
|
"thumbnails": thumbs_entry["thumbs"],
|
|
"thumbnails_more": thumbs_entry["more"],
|
|
"attachments": atts_map.get(post.id, []),
|
|
}
|