feat(ingest): localize inline post-body images to local copies (Phase 2)

Render a post body faithfully by serving our stored copies of inline
images instead of hotlinking the public CDN. The join key is the CDN
filehash (32-hex MD5) shared between a body <img src> and the media URL
we downloaded (the same identity extract_media dedups by):

- utils.paths.filehash_from_url — one source of truth for the extractor;
  patreon_client._filehash now delegates so capture- and render-time
  hashing cannot drift.
- ImageRecord gains source_url (provenance) + source_filehash (indexed
  match key); migration 0051.
- the per-media sidecar carries the file's source_url; the importer
  persists it (NULL-only) on the ImageRecord via _apply_sidecar.
- post_feed_service.get_post remaps body <img src> -> /images/<path> for
  every inline image whose filehash maps to a stored image of THIS
  artist; unmatched / pre-Phase-2 images keep hotlinking.

Pre-existing on-disk images have no filehash yet, so they fall back to
hotlinking until re-downloaded; localization is forward-looking.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/models/image_record.py

@@ -49,6 +49,18 @@ class ImageRecord(Base):
     # Thumbnail (populated by FC-2)
     thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
 
+    # Source provenance for downloaded media (#830 Phase 2). `source_url` is the
+    # CDN/origin URL the file was fetched from (debugging + future re-fetch).
+    # `source_filehash` is the URL's 32-hex CDN identity segment
+    # (utils.paths.filehash_from_url) — the JOIN KEY that maps a post body's
+    # inline `<img src=CDN>` back to this local copy so the rendered body serves
+    # our stored image instead of hotlinking the public source. Indexed for the
+    # render-time lookup. NULL for filesystem-imported / pre-Phase-2 rows.
+    source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
+    source_filehash: Mapped[str | None] = mapped_column(
+        String(32), nullable=True, index=True
+    )
+
     # Origin / provenance pointers
     origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
     primary_post_id: Mapped[int | None] = mapped_column(

backend/app/services/importer.py

@@ -35,6 +35,7 @@ from ..utils import safe_probe
 from ..utils.paths import (
     derive_subdir,
     derive_top_level_artist,
+    filehash_from_url,
     hash_suffixed_name,
     safe_ext,
 )
@@ -1046,6 +1047,14 @@ class Importer:
         if record.artist_id is None:
             record.artist_id = artist.id
 
+        # #830 Phase 2: persist the file's CDN source identity (NULL-only, so a
+        # re-import / enrich-on-duplicate never clobbers it) — the filehash is
+        # the join key that lets post_feed_service remap the body's inline
+        # `<img src=CDN>` to this local copy at render time.
+        if sd.source_url and record.source_filehash is None:
+            record.source_url = sd.source_url
+            record.source_filehash = filehash_from_url(sd.source_url)
+
         if explicit_source is not None:
             src = explicit_source
         else:

backend/app/services/patreon_client.py

@@ -39,7 +39,7 @@ from urllib.parse import parse_qs, urlsplit
 
 import requests
 
-from ..utils.paths import safe_ext
+from ..utils.paths import filehash_from_url, safe_ext
 
 log = logging.getLogger(__name__)
 
@@ -90,12 +90,6 @@ _FIELDS_POST = (
 _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
 _FIELDS_CAMPAIGN = "name,url"
 
-# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
-# Patreon's stable per-file identity and is what we dedup + ledger against.
-# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
-# in the URL (path or query); real Patreon CDN URLs carry exactly one.
-_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
-
 # Inline post `content` is HTML; images are emitted as <img ... src="...">.
 # Pull every src; downstream dedup collapses any that duplicate a gallery item
 # by filehash. Tolerant of attribute ordering and single/double quotes.
@@ -188,10 +182,9 @@ def _load_session(cookies_path: str | Path | None) -> requests.Session:
 
 
 def _filehash(url: str) -> str | None:
-    if not url:
-        return None
-    match = _FILEHASH_RE.search(url)
-    return match.group(1).lower() if match else None
+    # Delegate to the shared extractor (utils.paths) so capture-time persistence
+    # and render-time inline-image matching use the EXACT same identity.
+    return filehash_from_url(url)
 
 
 def _basename_from_url(url: str) -> str:

backend/app/services/patreon_downloader.py

@@ -294,7 +294,7 @@ class PatreonDownloader:
                     path=quarantine_dest, error=reason,
                 )
 
-        self._write_sidecar(post, out_path)
+        self._write_sidecar(post, out_path, source_url=media.url)
         return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
 
     # -- download seams ----------------------------------------------------
@@ -490,7 +490,9 @@ class PatreonDownloader:
 
     # -- sidecar -----------------------------------------------------------
 
-    def _write_sidecar(self, post: dict, media_path: Path) -> Path:
+    def _write_sidecar(
+        self, post: dict, media_path: Path, *, source_url: str | None = None
+    ) -> Path:
         """Write the importer-consumed sidecar next to `media_path`.
 
         Patreon uses base-default sidecar keys (parse_sidecar maps
@@ -498,13 +500,22 @@ class PatreonDownloader:
         content->description, published_at->post_date, url->post_url). Patreon
         registers no derive_post_url, so `url` is trusted as the permalink — we
         pass the post's attributes.url.
-        """
-        return self._write_sidecar_data(post, media_path.with_suffix(".json"))
 
-    def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path:
+        `source_url` is THIS file's CDN URL (#830 Phase 2). The importer persists
+        its filehash on the ImageRecord so the post body's inline `<img src>` can
+        be remapped to the local copy at render time.
+        """
+        return self._write_sidecar_data(
+            post, media_path.with_suffix(".json"), source_url=source_url
+        )
+
+    def _write_sidecar_data(
+        self, post: dict, sidecar_path: Path, *, source_url: str | None = None
+    ) -> Path:
         """Serialize the post's metadata to `sidecar_path`. Shared by the
         per-media sidecar (next to each file) and the post-only sidecar
-        (`write_post_record`, for media-less posts)."""
+        (`write_post_record`, for media-less posts). `source_url` is set only for
+        the per-media sidecar — a media-less post has no source file."""
         attrs = post.get("attributes") or {}
         title = attrs.get("title")
         content = attrs.get("content")
@@ -530,6 +541,8 @@ class PatreonDownloader:
             "published_at": published if isinstance(published, str) else None,
             "url": url if isinstance(url, str) else None,
         }
+        if source_url:
+            data["source_url"] = source_url
         sidecar_path.write_text(json.dumps(data, indent=2))
         return sidecar_path
 

backend/app/services/post_feed_service.py

@@ -11,6 +11,8 @@ attachments from PostAttachment) so the API layer can jsonify directly.
 """
 from __future__ import annotations
 
+from html import unescape
+
 from sqlalchemy import and_, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -23,9 +25,14 @@ from ..models import (
     PostAttachment,
     Source,
 )
-from ..utils.html_sanitize import sanitize_post_html
+from ..utils.html_sanitize import (
+    extract_img_srcs,
+    rewrite_img_srcs,
+    sanitize_post_html,
+)
+from ..utils.paths import filehash_from_url
 from ..utils.text import html_to_plain, truncate_at_word
-from .gallery_service import thumbnail_url
+from .gallery_service import image_url, thumbnail_url
 from .pagination import decode_cursor, encode_cursor
 
 DESCRIPTION_LIMIT = 280
@@ -195,11 +202,52 @@ class PostFeedService:
         item["description_full"] = html_to_plain(post.description)
         # Sanitized HTML body for faithful (semantic) rendering in the post view;
         # detail-only (the feed list stays lightweight plain text). None when the
-        # post has no body.
-        item["description_html"] = sanitize_post_html(post.description)
+        # post has no body. Inline `<img>` sources are remapped to locally-served
+        # copies (#830 Phase 2) so the body never hotlinks the public CDN.
+        item["description_html"] = await self._localize_inline_images(
+            sanitize_post_html(post.description), post.artist_id,
+        )
         item["external_links"] = await self._external_links_for(post.id)
         return item
 
+    async def _localize_inline_images(
+        self, html: str | None, artist_id: int | None,
+    ) -> str | None:
+        """Rewrite a post body's inline `<img src=CDN>` to locally-served copies.
+
+        The join key is the CDN filehash the downloader persisted on each
+        ImageRecord (source_filehash): for every body image whose filehash maps
+        to a stored image of THIS artist, swap the src to /images/<path>. Images
+        we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
+        they keep hotlinking, which is the prior behavior. Scoped to the post's
+        artist so one creator's body never resolves to another's file."""
+        if not html or artist_id is None:
+            return html
+        srcs = extract_img_srcs(html)
+        if not srcs:
+            return html
+        # filehash -> the raw (as-in-HTML) src strings carrying it. A body can
+        # repeat the same image; keep every raw form so each is substituted.
+        by_hash: dict[str, list[str]] = {}
+        for raw in srcs:
+            fh = filehash_from_url(unescape(raw))
+            if fh:
+                by_hash.setdefault(fh, []).append(raw)
+        if not by_hash:
+            return html
+        rows = (await self.session.execute(
+            select(ImageRecord.source_filehash, ImageRecord.path)
+            .where(
+                ImageRecord.artist_id == artist_id,
+                ImageRecord.source_filehash.in_(list(by_hash)),
+            )
+        )).all()
+        replace: dict[str, str] = {}
+        for fh, path in rows:
+            for raw in by_hash.get(fh, ()):
+                replace[raw] = image_url(path)
+        return rewrite_img_srcs(html, replace)
+
     async def _external_links_for(self, post_id: int) -> list[dict]:
         """Off-platform file-host links recorded for a post (detail-only). Each
         carries its host, full url, label, and download status so the post view

backend/app/utils/html_sanitize.py

@@ -6,6 +6,8 @@ server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used;
 bleach is deprecated upstream and intentionally not used.
 """
 
+import re
+
 import nh3
 
 # A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped
@@ -50,3 +52,39 @@ def sanitize_post_html(raw: str | None) -> str | None:
         link_rel="noopener noreferrer",
         clean_content_tags={"script", "style"},
     )
+
+
+# nh3 emits attributes double-quoted, so a single `src="..."` capture is enough
+# to read/rewrite inline-image sources on already-sanitized markup (#830 Phase 2).
+_IMG_SRC_RE = re.compile(r'(<img\b[^>]*?\bsrc=")([^"]*)(")', re.IGNORECASE)
+
+
+def extract_img_srcs(html: str | None) -> list[str]:
+    """The raw `src` values of every <img> in `html`, in document order, deduped
+    (first kept). Returned EXACTLY as they appear (still entity-escaped) so a
+    caller can match them back for verbatim substitution."""
+    if not html:
+        return []
+    out: list[str] = []
+    seen: set[str] = set()
+    for _pre, src, _post in _IMG_SRC_RE.findall(html):
+        if src and src not in seen:
+            seen.add(src)
+            out.append(src)
+    return out
+
+
+def rewrite_img_srcs(html: str | None, replace: dict[str, str]) -> str | None:
+    """Swap each <img src> whose raw (as-in-HTML) value is a key in `replace`
+    with the mapped value; others are left untouched. Used to point a post body's
+    inline images at locally-served copies (#830 Phase 2). Operates only on src
+    values of already-sanitized markup, so the result stays within the allowlist.
+    Returns `html` unchanged when it or `replace` is empty."""
+    if not html or not replace:
+        return html
+
+    def _sub(m: re.Match) -> str:
+        pre, src, post = m.group(1), m.group(2), m.group(3)
+        return f"{pre}{replace.get(src, src)}{post}"
+
+    return _IMG_SRC_RE.sub(_sub, html)

backend/app/utils/paths.py

@@ -1,9 +1,28 @@
 """Filesystem path helpers — destination derivation, hash-suffixed names."""
 
+import re
 from pathlib import Path
 
 _MAX_EXT_LEN = 16
 
+# A Patreon/gallery CDN URL embeds a 32-char hex (MD5) path segment that is the
+# file's stable per-file identity — the same role gallery-dl's `_filehash`
+# plays. It is the join key between a post body `<img src=CDN>` and the local
+# copy we downloaded (extract_media dedups content vs gallery images by it), so
+# this ONE extractor must be used for both capture-time persistence and
+# render-time matching — they cannot be allowed to drift. Match the FIRST 32-hex
+# run anywhere in the URL (path or query); real CDN URLs carry exactly one.
+_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
+
+
+def filehash_from_url(url: str | None) -> str | None:
+    """The 32-char hex (MD5) CDN identity segment of `url`, lowercased, or None
+    when the URL is empty / carries no such segment."""
+    if not url:
+        return None
+    match = _FILEHASH_RE.search(url)
+    return match.group(1).lower() if match else None
+
 
 def safe_ext(name: str | Path) -> str:
     """Conservatively extract a short, alphanumeric file extension.

backend/app/utils/sidecar.py

@@ -27,6 +27,11 @@ class SidecarData:
     description: str | None
     attachment_count: int | None
     post_date: datetime | None
+    # Per-FILE CDN/origin URL the media was downloaded from (#830 Phase 2). Only
+    # the per-media sidecar carries it; post-only sidecars leave it None. The
+    # importer persists it on the ImageRecord so the body's inline `<img src>`
+    # can be matched (by filehash) to the local copy at render time.
+    source_url: str | None
     raw: dict
 
 
@@ -172,6 +177,11 @@ def parse_sidecar(data: dict) -> SidecarData:
     else:
         post_url = _first_str(data, ("url", "post_url"))
 
+    # Per-file source URL (#830 Phase 2): the native downloader writes it into
+    # each media sidecar. `url` is the post permalink for Patreon (handled
+    # above), so source_url has its own dedicated key and never collides.
+    source_url = _first_str(data, ("source_url",))
+
     return SidecarData(
         platform=platform,
         external_post_id=external_post_id,
@@ -180,5 +190,6 @@ def parse_sidecar(data: dict) -> SidecarData:
         description=description,
         attachment_count=attachment_count,
         post_date=post_date,
+        source_url=source_url,
         raw=data,
     )