diff --git a/alembic/versions/0051_image_source_provenance.py b/alembic/versions/0051_image_source_provenance.py
new file mode 100644
index 0000000..595077d
--- /dev/null
+++ b/alembic/versions/0051_image_source_provenance.py
@@ -0,0 +1,38 @@
+"""image_record: source_url + source_filehash (inline-image localization)
+
+#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline
+images instead of hotlinking the public CDN. The join key between a body
+`
` and the local file is the CDN's 32-hex filehash (the same
+identity extract_media dedups by). Persist it (indexed) plus the full source
+URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing
+rows — those fall back to hotlinking until re-downloaded.
+
+Revision ID: 0051
+Revises: 0050
+Create Date: 2026-06-14
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0051"
+down_revision: Union[str, None] = "0050"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True))
+ op.add_column(
+ "image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True)
+ )
+ op.create_index(
+ "ix_image_record_source_filehash", "image_record", ["source_filehash"]
+ )
+
+
+def downgrade() -> None:
+ op.drop_index("ix_image_record_source_filehash", table_name="image_record")
+ op.drop_column("image_record", "source_filehash")
+ op.drop_column("image_record", "source_url")
diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py
index ac57c9a..07cb0ec 100644
--- a/backend/app/models/image_record.py
+++ b/backend/app/models/image_record.py
@@ -49,6 +49,18 @@ class ImageRecord(Base):
# Thumbnail (populated by FC-2)
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
+ # Source provenance for downloaded media (#830 Phase 2). `source_url` is the
+ # CDN/origin URL the file was fetched from (debugging + future re-fetch).
+ # `source_filehash` is the URL's 32-hex CDN identity segment
+ # (utils.paths.filehash_from_url) — the JOIN KEY that maps a post body's
+ # inline `` back to this local copy so the rendered body serves
+ # our stored image instead of hotlinking the public source. Indexed for the
+ # render-time lookup. NULL for filesystem-imported / pre-Phase-2 rows.
+ source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
+ source_filehash: Mapped[str | None] = mapped_column(
+ String(32), nullable=True, index=True
+ )
+
# Origin / provenance pointers
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
primary_post_id: Mapped[int | None] = mapped_column(
diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py
index d81ba34..c9b5abe 100644
--- a/backend/app/services/importer.py
+++ b/backend/app/services/importer.py
@@ -35,6 +35,7 @@ from ..utils import safe_probe
from ..utils.paths import (
derive_subdir,
derive_top_level_artist,
+ filehash_from_url,
hash_suffixed_name,
safe_ext,
)
@@ -1046,6 +1047,14 @@ class Importer:
if record.artist_id is None:
record.artist_id = artist.id
+ # #830 Phase 2: persist the file's CDN source identity (NULL-only, so a
+ # re-import / enrich-on-duplicate never clobbers it) — the filehash is
+ # the join key that lets post_feed_service remap the body's inline
+ # `` to this local copy at render time.
+ if sd.source_url and record.source_filehash is None:
+ record.source_url = sd.source_url
+ record.source_filehash = filehash_from_url(sd.source_url)
+
if explicit_source is not None:
src = explicit_source
else:
diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py
index af56aca..f79f627 100644
--- a/backend/app/services/patreon_client.py
+++ b/backend/app/services/patreon_client.py
@@ -39,7 +39,7 @@ from urllib.parse import parse_qs, urlsplit
import requests
-from ..utils.paths import safe_ext
+from ..utils.paths import filehash_from_url, safe_ext
log = logging.getLogger(__name__)
@@ -90,12 +90,6 @@ _FIELDS_POST = (
_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
_FIELDS_CAMPAIGN = "name,url"
-# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
-# Patreon's stable per-file identity and is what we dedup + ledger against.
-# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
-# in the URL (path or query); real Patreon CDN URLs carry exactly one.
-_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
-
# Inline post `content` is HTML; images are emitted as 
.
# Pull every src; downstream dedup collapses any that duplicate a gallery item
# by filehash. Tolerant of attribute ordering and single/double quotes.
@@ -188,10 +182,9 @@ def _load_session(cookies_path: str | Path | None) -> requests.Session:
def _filehash(url: str) -> str | None:
- if not url:
- return None
- match = _FILEHASH_RE.search(url)
- return match.group(1).lower() if match else None
+ # Delegate to the shared extractor (utils.paths) so capture-time persistence
+ # and render-time inline-image matching use the EXACT same identity.
+ return filehash_from_url(url)
def _basename_from_url(url: str) -> str:
diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py
index 42540cb..1f7443d 100644
--- a/backend/app/services/patreon_downloader.py
+++ b/backend/app/services/patreon_downloader.py
@@ -294,7 +294,7 @@ class PatreonDownloader:
path=quarantine_dest, error=reason,
)
- self._write_sidecar(post, out_path)
+ self._write_sidecar(post, out_path, source_url=media.url)
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
# -- download seams ----------------------------------------------------
@@ -490,7 +490,9 @@ class PatreonDownloader:
# -- sidecar -----------------------------------------------------------
- def _write_sidecar(self, post: dict, media_path: Path) -> Path:
+ def _write_sidecar(
+ self, post: dict, media_path: Path, *, source_url: str | None = None
+ ) -> Path:
"""Write the importer-consumed sidecar next to `media_path`.
Patreon uses base-default sidecar keys (parse_sidecar maps
@@ -498,13 +500,22 @@ class PatreonDownloader:
content->description, published_at->post_date, url->post_url). Patreon
registers no derive_post_url, so `url` is trusted as the permalink — we
pass the post's attributes.url.
- """
- return self._write_sidecar_data(post, media_path.with_suffix(".json"))
- def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path:
+ `source_url` is THIS file's CDN URL (#830 Phase 2). The importer persists
+ its filehash on the ImageRecord so the post body's inline `![]()
` can
+ be remapped to the local copy at render time.
+ """
+ return self._write_sidecar_data(
+ post, media_path.with_suffix(".json"), source_url=source_url
+ )
+
+ def _write_sidecar_data(
+ self, post: dict, sidecar_path: Path, *, source_url: str | None = None
+ ) -> Path:
"""Serialize the post's metadata to `sidecar_path`. Shared by the
per-media sidecar (next to each file) and the post-only sidecar
- (`write_post_record`, for media-less posts)."""
+ (`write_post_record`, for media-less posts). `source_url` is set only for
+ the per-media sidecar — a media-less post has no source file."""
attrs = post.get("attributes") or {}
title = attrs.get("title")
content = attrs.get("content")
@@ -530,6 +541,8 @@ class PatreonDownloader:
"published_at": published if isinstance(published, str) else None,
"url": url if isinstance(url, str) else None,
}
+ if source_url:
+ data["source_url"] = source_url
sidecar_path.write_text(json.dumps(data, indent=2))
return sidecar_path
diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py
index ad053d4..d21f19d 100644
--- a/backend/app/services/post_feed_service.py
+++ b/backend/app/services/post_feed_service.py
@@ -11,6 +11,8 @@ attachments from PostAttachment) so the API layer can jsonify directly.
"""
from __future__ import annotations
+from html import unescape
+
from sqlalchemy import and_, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -23,9 +25,14 @@ from ..models import (
PostAttachment,
Source,
)
-from ..utils.html_sanitize import sanitize_post_html
+from ..utils.html_sanitize import (
+ extract_img_srcs,
+ rewrite_img_srcs,
+ sanitize_post_html,
+)
+from ..utils.paths import filehash_from_url
from ..utils.text import html_to_plain, truncate_at_word
-from .gallery_service import thumbnail_url
+from .gallery_service import image_url, thumbnail_url
from .pagination import decode_cursor, encode_cursor
DESCRIPTION_LIMIT = 280
@@ -195,11 +202,52 @@ class PostFeedService:
item["description_full"] = html_to_plain(post.description)
# Sanitized HTML body for faithful (semantic) rendering in the post view;
# detail-only (the feed list stays lightweight plain text). None when the
- # post has no body.
- item["description_html"] = sanitize_post_html(post.description)
+ # post has no body. Inline `![]()
` sources are remapped to locally-served
+ # copies (#830 Phase 2) so the body never hotlinks the public CDN.
+ item["description_html"] = await self._localize_inline_images(
+ sanitize_post_html(post.description), post.artist_id,
+ )
item["external_links"] = await self._external_links_for(post.id)
return item
+ async def _localize_inline_images(
+ self, html: str | None, artist_id: int | None,
+ ) -> str | None:
+ """Rewrite a post body's inline `` to locally-served copies.
+
+ The join key is the CDN filehash the downloader persisted on each
+ ImageRecord (source_filehash): for every body image whose filehash maps
+ to a stored image of THIS artist, swap the src to /images/. Images
+ we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
+ they keep hotlinking, which is the prior behavior. Scoped to the post's
+ artist so one creator's body never resolves to another's file."""
+ if not html or artist_id is None:
+ return html
+ srcs = extract_img_srcs(html)
+ if not srcs:
+ return html
+ # filehash -> the raw (as-in-HTML) src strings carrying it. A body can
+ # repeat the same image; keep every raw form so each is substituted.
+ by_hash: dict[str, list[str]] = {}
+ for raw in srcs:
+ fh = filehash_from_url(unescape(raw))
+ if fh:
+ by_hash.setdefault(fh, []).append(raw)
+ if not by_hash:
+ return html
+ rows = (await self.session.execute(
+ select(ImageRecord.source_filehash, ImageRecord.path)
+ .where(
+ ImageRecord.artist_id == artist_id,
+ ImageRecord.source_filehash.in_(list(by_hash)),
+ )
+ )).all()
+ replace: dict[str, str] = {}
+ for fh, path in rows:
+ for raw in by_hash.get(fh, ()):
+ replace[raw] = image_url(path)
+ return rewrite_img_srcs(html, replace)
+
async def _external_links_for(self, post_id: int) -> list[dict]:
"""Off-platform file-host links recorded for a post (detail-only). Each
carries its host, full url, label, and download status so the post view
diff --git a/backend/app/utils/html_sanitize.py b/backend/app/utils/html_sanitize.py
index 5748a5e..363dd77 100644
--- a/backend/app/utils/html_sanitize.py
+++ b/backend/app/utils/html_sanitize.py
@@ -6,6 +6,8 @@ server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used;
bleach is deprecated upstream and intentionally not used.
"""
+import re
+
import nh3
# A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped
@@ -50,3 +52,39 @@ def sanitize_post_html(raw: str | None) -> str | None:
link_rel="noopener noreferrer",
clean_content_tags={"script", "style"},
)
+
+
+# nh3 emits attributes double-quoted, so a single `src="..."` capture is enough
+# to read/rewrite inline-image sources on already-sanitized markup (#830 Phase 2).
+_IMG_SRC_RE = re.compile(r'(]*?\bsrc=")([^"]*)(")', re.IGNORECASE)
+
+
+def extract_img_srcs(html: str | None) -> list[str]:
+ """The raw `src` values of every ![]()
in `html`, in document order, deduped
+ (first kept). Returned EXACTLY as they appear (still entity-escaped) so a
+ caller can match them back for verbatim substitution."""
+ if not html:
+ return []
+ out: list[str] = []
+ seen: set[str] = set()
+ for _pre, src, _post in _IMG_SRC_RE.findall(html):
+ if src and src not in seen:
+ seen.add(src)
+ out.append(src)
+ return out
+
+
+def rewrite_img_srcs(html: str | None, replace: dict[str, str]) -> str | None:
+ """Swap each ![]()
whose raw (as-in-HTML) value is a key in `replace`
+ with the mapped value; others are left untouched. Used to point a post body's
+ inline images at locally-served copies (#830 Phase 2). Operates only on src
+ values of already-sanitized markup, so the result stays within the allowlist.
+ Returns `html` unchanged when it or `replace` is empty."""
+ if not html or not replace:
+ return html
+
+ def _sub(m: re.Match) -> str:
+ pre, src, post = m.group(1), m.group(2), m.group(3)
+ return f"{pre}{replace.get(src, src)}{post}"
+
+ return _IMG_SRC_RE.sub(_sub, html)
diff --git a/backend/app/utils/paths.py b/backend/app/utils/paths.py
index d9bc48e..7130e90 100644
--- a/backend/app/utils/paths.py
+++ b/backend/app/utils/paths.py
@@ -1,9 +1,28 @@
"""Filesystem path helpers — destination derivation, hash-suffixed names."""
+import re
from pathlib import Path
_MAX_EXT_LEN = 16
+# A Patreon/gallery CDN URL embeds a 32-char hex (MD5) path segment that is the
+# file's stable per-file identity — the same role gallery-dl's `_filehash`
+# plays. It is the join key between a post body `` and the local
+# copy we downloaded (extract_media dedups content vs gallery images by it), so
+# this ONE extractor must be used for both capture-time persistence and
+# render-time matching — they cannot be allowed to drift. Match the FIRST 32-hex
+# run anywhere in the URL (path or query); real CDN URLs carry exactly one.
+_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
+
+
+def filehash_from_url(url: str | None) -> str | None:
+ """The 32-char hex (MD5) CDN identity segment of `url`, lowercased, or None
+ when the URL is empty / carries no such segment."""
+ if not url:
+ return None
+ match = _FILEHASH_RE.search(url)
+ return match.group(1).lower() if match else None
+
def safe_ext(name: str | Path) -> str:
"""Conservatively extract a short, alphanumeric file extension.
diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py
index 0d81930..1e2a7d5 100644
--- a/backend/app/utils/sidecar.py
+++ b/backend/app/utils/sidecar.py
@@ -27,6 +27,11 @@ class SidecarData:
description: str | None
attachment_count: int | None
post_date: datetime | None
+ # Per-FILE CDN/origin URL the media was downloaded from (#830 Phase 2). Only
+ # the per-media sidecar carries it; post-only sidecars leave it None. The
+ # importer persists it on the ImageRecord so the body's inline `![]()
`
+ # can be matched (by filehash) to the local copy at render time.
+ source_url: str | None
raw: dict
@@ -172,6 +177,11 @@ def parse_sidecar(data: dict) -> SidecarData:
else:
post_url = _first_str(data, ("url", "post_url"))
+ # Per-file source URL (#830 Phase 2): the native downloader writes it into
+ # each media sidecar. `url` is the post permalink for Patreon (handled
+ # above), so source_url has its own dedicated key and never collides.
+ source_url = _first_str(data, ("source_url",))
+
return SidecarData(
platform=platform,
external_post_id=external_post_id,
@@ -180,5 +190,6 @@ def parse_sidecar(data: dict) -> SidecarData:
description=description,
attachment_count=attachment_count,
post_date=post_date,
+ source_url=source_url,
raw=data,
)
diff --git a/tests/test_html_sanitize.py b/tests/test_html_sanitize.py
index 63eeeee..5d92dae 100644
--- a/tests/test_html_sanitize.py
+++ b/tests/test_html_sanitize.py
@@ -1,4 +1,8 @@
-from backend.app.utils.html_sanitize import sanitize_post_html
+from backend.app.utils.html_sanitize import (
+ extract_img_srcs,
+ rewrite_img_srcs,
+ sanitize_post_html,
+)
def test_none_and_blank_return_none():
@@ -52,3 +56,40 @@ def test_headings_lists_and_images_survive():
def test_image_javascript_src_dropped():
out = sanitize_post_html(')
')
assert "javascript:" not in out
+
+
+# --- inline-image localization helpers (#830 Phase 2) ----------------------
+
+
+def test_extract_img_srcs_in_order_deduped():
+ html = (
+ 'a
'
+ '
'
+ ''
+ )
+ assert extract_img_srcs(html) == [
+ "https://cdn.test/1.jpg",
+ "https://cdn.test/2.jpg",
+ ]
+
+
+def test_extract_img_srcs_empty():
+ assert extract_img_srcs(None) == []
+ assert extract_img_srcs("no images
") == []
+
+
+def test_rewrite_img_srcs_swaps_only_mapped():
+ html = (
+ ''
+ ''
+ )
+ out = rewrite_img_srcs(html, {"https://cdn.test/1.jpg": "/images/local/1.jpg"})
+ assert 'src="/images/local/1.jpg"' in out
+ assert 'alt="a"' in out # other attributes preserved
+ assert 'src="https://cdn.test/2.jpg"' in out # unmapped left alone
+
+
+def test_rewrite_img_srcs_noop_on_empty():
+ html = ''
+ assert rewrite_img_srcs(html, {}) == html
+ assert rewrite_img_srcs(None, {"a": "b"}) is None
diff --git a/tests/test_paths.py b/tests/test_paths.py
index 2976d1e..4b52f97 100644
--- a/tests/test_paths.py
+++ b/tests/test_paths.py
@@ -3,12 +3,31 @@ from pathlib import Path
from backend.app.utils.paths import (
derive_subdir,
derive_top_level_artist,
+ filehash_from_url,
hash_suffixed_name,
)
IMPORT = Path("/import")
+def test_filehash_from_url_extracts_lowercased_md5():
+ url = "https://c10.patreonusercontent.com/4/patreon-media/p/post/1/AbC123DeF456789012345678901234EF/1/img.png?token=x"
+ assert filehash_from_url(url) == "abc123def456789012345678901234ef"
+
+
+def test_filehash_from_url_none_and_no_hash():
+ assert filehash_from_url(None) is None
+ assert filehash_from_url("") is None
+ assert filehash_from_url("https://example.test/a.png") is None
+
+
+def test_filehash_from_url_query_unescaping_irrelevant():
+ # The hash sits in the path before the query, so entity-escaped ampersands
+ # in the query never affect the match (render-time matches escaped srcs).
+ base = "https://cdn.test/p/0123456789abcdef0123456789ABCDEF/x.jpg"
+ assert filehash_from_url(base + "?a=1&b=2") == "0123456789abcdef0123456789abcdef"
+
+
def test_derive_subdir_nested():
assert derive_subdir(Path("/import/Alice/sub/x.png"), IMPORT) == "Alice/sub"
diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py
index 3a1d15f..06d4700 100644
--- a/tests/test_patreon_downloader.py
+++ b/tests/test_patreon_downloader.py
@@ -175,6 +175,9 @@ def test_sidecar_written_and_findable(tmp_path):
assert data["id"] == "1001"
assert data["title"] == "My Post Title"
assert data["url"] == "https://www.patreon.com/posts/1001"
+ # #830 Phase 2: the per-media sidecar records THIS file's CDN URL so the
+ # importer can persist its filehash for inline-image localization.
+ assert data["source_url"] == "https://cdn.patreon.com/media1.png"
def test_skip_seen(tmp_path):
diff --git a/tests/test_post_feed_service.py b/tests/test_post_feed_service.py
index 7374743..78a01da 100644
--- a/tests/test_post_feed_service.py
+++ b/tests/test_post_feed_service.py
@@ -438,6 +438,62 @@ async def test_get_post_returns_sanitized_html_body(db):
assert "