feat(ingest): localize inline post-body images to local copies (Phase 2)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 22s
CI / backend-lint-and-test (push) Successful in 38s
CI / integration (push) Successful in 3m14s

Render a post body faithfully by serving our stored copies of inline
images instead of hotlinking the public CDN. The join key is the CDN
filehash (32-hex MD5) shared between a body <img src> and the media URL
we downloaded (the same identity extract_media dedups by):

- utils.paths.filehash_from_url — one source of truth for the extractor;
  patreon_client._filehash now delegates so capture- and render-time
  hashing cannot drift.
- ImageRecord gains source_url (provenance) + source_filehash (indexed
  match key); migration 0051.
- the per-media sidecar carries the file's source_url; the importer
  persists it (NULL-only) on the ImageRecord via _apply_sidecar.
- post_feed_service.get_post remaps body <img src> -> /images/<path> for
  every inline image whose filehash maps to a stored image of THIS
  artist; unmatched / pre-Phase-2 images keep hotlinking.

Pre-existing on-disk images have no filehash yet, so they fall back to
hotlinking until re-downloaded; localization is forward-looking.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-14 16:39:58 -04:00
parent 5e1655384f
commit 96c29c370b
15 changed files with 350 additions and 22 deletions
+52 -4
View File
@@ -11,6 +11,8 @@ attachments from PostAttachment) so the API layer can jsonify directly.
"""
from __future__ import annotations
from html import unescape
from sqlalchemy import and_, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -23,9 +25,14 @@ from ..models import (
PostAttachment,
Source,
)
from ..utils.html_sanitize import sanitize_post_html
from ..utils.html_sanitize import (
extract_img_srcs,
rewrite_img_srcs,
sanitize_post_html,
)
from ..utils.paths import filehash_from_url
from ..utils.text import html_to_plain, truncate_at_word
from .gallery_service import thumbnail_url
from .gallery_service import image_url, thumbnail_url
from .pagination import decode_cursor, encode_cursor
DESCRIPTION_LIMIT = 280
@@ -195,11 +202,52 @@ class PostFeedService:
item["description_full"] = html_to_plain(post.description)
# Sanitized HTML body for faithful (semantic) rendering in the post view;
# detail-only (the feed list stays lightweight plain text). None when the
# post has no body.
item["description_html"] = sanitize_post_html(post.description)
# post has no body. Inline `<img>` sources are remapped to locally-served
# copies (#830 Phase 2) so the body never hotlinks the public CDN.
item["description_html"] = await self._localize_inline_images(
sanitize_post_html(post.description), post.artist_id,
)
item["external_links"] = await self._external_links_for(post.id)
return item
async def _localize_inline_images(
self, html: str | None, artist_id: int | None,
) -> str | None:
"""Rewrite a post body's inline `<img src=CDN>` to locally-served copies.
The join key is the CDN filehash the downloader persisted on each
ImageRecord (source_filehash): for every body image whose filehash maps
to a stored image of THIS artist, swap the src to /images/<path>. Images
we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
they keep hotlinking, which is the prior behavior. Scoped to the post's
artist so one creator's body never resolves to another's file."""
if not html or artist_id is None:
return html
srcs = extract_img_srcs(html)
if not srcs:
return html
# filehash -> the raw (as-in-HTML) src strings carrying it. A body can
# repeat the same image; keep every raw form so each is substituted.
by_hash: dict[str, list[str]] = {}
for raw in srcs:
fh = filehash_from_url(unescape(raw))
if fh:
by_hash.setdefault(fh, []).append(raw)
if not by_hash:
return html
rows = (await self.session.execute(
select(ImageRecord.source_filehash, ImageRecord.path)
.where(
ImageRecord.artist_id == artist_id,
ImageRecord.source_filehash.in_(list(by_hash)),
)
)).all()
replace: dict[str, str] = {}
for fh, path in rows:
for raw in by_hash.get(fh, ()):
replace[raw] = image_url(path)
return rewrite_img_srcs(html, replace)
async def _external_links_for(self, post_id: int) -> list[dict]:
"""Off-platform file-host links recorded for a post (detail-only). Each
carries its host, full url, label, and download status so the post view