feat(ingest): localize inline post-body images to local copies (Phase 2)
Render a post body faithfully by serving our stored copies of inline images instead of hotlinking the public CDN. The join key is the CDN filehash (32-hex MD5) shared between a body <img src> and the media URL we downloaded (the same identity extract_media dedups by): - utils.paths.filehash_from_url — one source of truth for the extractor; patreon_client._filehash now delegates so capture- and render-time hashing cannot drift. - ImageRecord gains source_url (provenance) + source_filehash (indexed match key); migration 0051. - the per-media sidecar carries the file's source_url; the importer persists it (NULL-only) on the ImageRecord via _apply_sidecar. - post_feed_service.get_post remaps body <img src> -> /images/<path> for every inline image whose filehash maps to a stored image of THIS artist; unmatched / pre-Phase-2 images keep hotlinking. Pre-existing on-disk images have no filehash yet, so they fall back to hotlinking until re-downloaded; localization is forward-looking. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,8 @@ attachments from PostAttachment) so the API layer can jsonify directly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from html import unescape
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
@@ -23,9 +25,14 @@ from ..models import (
|
||||
PostAttachment,
|
||||
Source,
|
||||
)
|
||||
from ..utils.html_sanitize import sanitize_post_html
|
||||
from ..utils.html_sanitize import (
|
||||
extract_img_srcs,
|
||||
rewrite_img_srcs,
|
||||
sanitize_post_html,
|
||||
)
|
||||
from ..utils.paths import filehash_from_url
|
||||
from ..utils.text import html_to_plain, truncate_at_word
|
||||
from .gallery_service import thumbnail_url
|
||||
from .gallery_service import image_url, thumbnail_url
|
||||
from .pagination import decode_cursor, encode_cursor
|
||||
|
||||
DESCRIPTION_LIMIT = 280
|
||||
@@ -195,11 +202,52 @@ class PostFeedService:
|
||||
item["description_full"] = html_to_plain(post.description)
|
||||
# Sanitized HTML body for faithful (semantic) rendering in the post view;
|
||||
# detail-only (the feed list stays lightweight plain text). None when the
|
||||
# post has no body.
|
||||
item["description_html"] = sanitize_post_html(post.description)
|
||||
# post has no body. Inline `<img>` sources are remapped to locally-served
|
||||
# copies (#830 Phase 2) so the body never hotlinks the public CDN.
|
||||
item["description_html"] = await self._localize_inline_images(
|
||||
sanitize_post_html(post.description), post.artist_id,
|
||||
)
|
||||
item["external_links"] = await self._external_links_for(post.id)
|
||||
return item
|
||||
|
||||
async def _localize_inline_images(
|
||||
self, html: str | None, artist_id: int | None,
|
||||
) -> str | None:
|
||||
"""Rewrite a post body's inline `<img src=CDN>` to locally-served copies.
|
||||
|
||||
The join key is the CDN filehash the downloader persisted on each
|
||||
ImageRecord (source_filehash): for every body image whose filehash maps
|
||||
to a stored image of THIS artist, swap the src to /images/<path>. Images
|
||||
we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
|
||||
they keep hotlinking, which is the prior behavior. Scoped to the post's
|
||||
artist so one creator's body never resolves to another's file."""
|
||||
if not html or artist_id is None:
|
||||
return html
|
||||
srcs = extract_img_srcs(html)
|
||||
if not srcs:
|
||||
return html
|
||||
# filehash -> the raw (as-in-HTML) src strings carrying it. A body can
|
||||
# repeat the same image; keep every raw form so each is substituted.
|
||||
by_hash: dict[str, list[str]] = {}
|
||||
for raw in srcs:
|
||||
fh = filehash_from_url(unescape(raw))
|
||||
if fh:
|
||||
by_hash.setdefault(fh, []).append(raw)
|
||||
if not by_hash:
|
||||
return html
|
||||
rows = (await self.session.execute(
|
||||
select(ImageRecord.source_filehash, ImageRecord.path)
|
||||
.where(
|
||||
ImageRecord.artist_id == artist_id,
|
||||
ImageRecord.source_filehash.in_(list(by_hash)),
|
||||
)
|
||||
)).all()
|
||||
replace: dict[str, str] = {}
|
||||
for fh, path in rows:
|
||||
for raw in by_hash.get(fh, ()):
|
||||
replace[raw] = image_url(path)
|
||||
return rewrite_img_srcs(html, replace)
|
||||
|
||||
async def _external_links_for(self, post_id: int) -> list[dict]:
|
||||
"""Off-platform file-host links recorded for a post (detail-only). Each
|
||||
carries its host, full url, label, and download status so the post view
|
||||
|
||||
Reference in New Issue
Block a user