feat(ingest): localize inline post-body images to local copies (Phase 2)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 22s
CI / backend-lint-and-test (push) Successful in 38s
CI / integration (push) Successful in 3m14s

Render a post body faithfully by serving our stored copies of inline
images instead of hotlinking the public CDN. The join key is the CDN
filehash (32-hex MD5) shared between a body <img src> and the media URL
we downloaded (the same identity extract_media dedups by):

- utils.paths.filehash_from_url — one source of truth for the extractor;
  patreon_client._filehash now delegates so capture- and render-time
  hashing cannot drift.
- ImageRecord gains source_url (provenance) + source_filehash (indexed
  match key); migration 0051.
- the per-media sidecar carries the file's source_url; the importer
  persists it (NULL-only) on the ImageRecord via _apply_sidecar.
- post_feed_service.get_post remaps body <img src> -> /images/<path> for
  every inline image whose filehash maps to a stored image of THIS
  artist; unmatched / pre-Phase-2 images keep hotlinking.

Pre-existing on-disk images have no filehash yet, so they fall back to
hotlinking until re-downloaded; localization is forward-looking.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-14 16:39:58 -04:00
parent 5e1655384f
commit 96c29c370b
15 changed files with 350 additions and 22 deletions
@@ -0,0 +1,38 @@
"""image_record: source_url + source_filehash (inline-image localization)
#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline
images instead of hotlinking the public CDN. The join key between a body
`<img src=CDN>` and the local file is the CDN's 32-hex filehash (the same
identity extract_media dedups by). Persist it (indexed) plus the full source
URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing
rows — those fall back to hotlinking until re-downloaded.
Revision ID: 0051
Revises: 0050
Create Date: 2026-06-14
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0051"
down_revision: Union[str, None] = "0050"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True))
op.add_column(
"image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True)
)
op.create_index(
"ix_image_record_source_filehash", "image_record", ["source_filehash"]
)
def downgrade() -> None:
op.drop_index("ix_image_record_source_filehash", table_name="image_record")
op.drop_column("image_record", "source_filehash")
op.drop_column("image_record", "source_url")
+12
View File
@@ -49,6 +49,18 @@ class ImageRecord(Base):
# Thumbnail (populated by FC-2)
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
# Source provenance for downloaded media (#830 Phase 2). `source_url` is the
# CDN/origin URL the file was fetched from (debugging + future re-fetch).
# `source_filehash` is the URL's 32-hex CDN identity segment
# (utils.paths.filehash_from_url) — the JOIN KEY that maps a post body's
# inline `<img src=CDN>` back to this local copy so the rendered body serves
# our stored image instead of hotlinking the public source. Indexed for the
# render-time lookup. NULL for filesystem-imported / pre-Phase-2 rows.
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
source_filehash: Mapped[str | None] = mapped_column(
String(32), nullable=True, index=True
)
# Origin / provenance pointers
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
primary_post_id: Mapped[int | None] = mapped_column(
+9
View File
@@ -35,6 +35,7 @@ from ..utils import safe_probe
from ..utils.paths import (
derive_subdir,
derive_top_level_artist,
filehash_from_url,
hash_suffixed_name,
safe_ext,
)
@@ -1046,6 +1047,14 @@ class Importer:
if record.artist_id is None:
record.artist_id = artist.id
# #830 Phase 2: persist the file's CDN source identity (NULL-only, so a
# re-import / enrich-on-duplicate never clobbers it) — the filehash is
# the join key that lets post_feed_service remap the body's inline
# `<img src=CDN>` to this local copy at render time.
if sd.source_url and record.source_filehash is None:
record.source_url = sd.source_url
record.source_filehash = filehash_from_url(sd.source_url)
if explicit_source is not None:
src = explicit_source
else:
+4 -11
View File
@@ -39,7 +39,7 @@ from urllib.parse import parse_qs, urlsplit
import requests
from ..utils.paths import safe_ext
from ..utils.paths import filehash_from_url, safe_ext
log = logging.getLogger(__name__)
@@ -90,12 +90,6 @@ _FIELDS_POST = (
_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
_FIELDS_CAMPAIGN = "name,url"
# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
# Patreon's stable per-file identity and is what we dedup + ledger against.
# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
# in the URL (path or query); real Patreon CDN URLs carry exactly one.
_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
# Inline post `content` is HTML; images are emitted as <img ... src="...">.
# Pull every src; downstream dedup collapses any that duplicate a gallery item
# by filehash. Tolerant of attribute ordering and single/double quotes.
@@ -188,10 +182,9 @@ def _load_session(cookies_path: str | Path | None) -> requests.Session:
def _filehash(url: str) -> str | None:
if not url:
return None
match = _FILEHASH_RE.search(url)
return match.group(1).lower() if match else None
# Delegate to the shared extractor (utils.paths) so capture-time persistence
# and render-time inline-image matching use the EXACT same identity.
return filehash_from_url(url)
def _basename_from_url(url: str) -> str:
+19 -6
View File
@@ -294,7 +294,7 @@ class PatreonDownloader:
path=quarantine_dest, error=reason,
)
self._write_sidecar(post, out_path)
self._write_sidecar(post, out_path, source_url=media.url)
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
# -- download seams ----------------------------------------------------
@@ -490,7 +490,9 @@ class PatreonDownloader:
# -- sidecar -----------------------------------------------------------
def _write_sidecar(self, post: dict, media_path: Path) -> Path:
def _write_sidecar(
self, post: dict, media_path: Path, *, source_url: str | None = None
) -> Path:
"""Write the importer-consumed sidecar next to `media_path`.
Patreon uses base-default sidecar keys (parse_sidecar maps
@@ -498,13 +500,22 @@ class PatreonDownloader:
content->description, published_at->post_date, url->post_url). Patreon
registers no derive_post_url, so `url` is trusted as the permalink — we
pass the post's attributes.url.
"""
return self._write_sidecar_data(post, media_path.with_suffix(".json"))
def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path:
`source_url` is THIS file's CDN URL (#830 Phase 2). The importer persists
its filehash on the ImageRecord so the post body's inline `<img src>` can
be remapped to the local copy at render time.
"""
return self._write_sidecar_data(
post, media_path.with_suffix(".json"), source_url=source_url
)
def _write_sidecar_data(
self, post: dict, sidecar_path: Path, *, source_url: str | None = None
) -> Path:
"""Serialize the post's metadata to `sidecar_path`. Shared by the
per-media sidecar (next to each file) and the post-only sidecar
(`write_post_record`, for media-less posts)."""
(`write_post_record`, for media-less posts). `source_url` is set only for
the per-media sidecar — a media-less post has no source file."""
attrs = post.get("attributes") or {}
title = attrs.get("title")
content = attrs.get("content")
@@ -530,6 +541,8 @@ class PatreonDownloader:
"published_at": published if isinstance(published, str) else None,
"url": url if isinstance(url, str) else None,
}
if source_url:
data["source_url"] = source_url
sidecar_path.write_text(json.dumps(data, indent=2))
return sidecar_path
+52 -4
View File
@@ -11,6 +11,8 @@ attachments from PostAttachment) so the API layer can jsonify directly.
"""
from __future__ import annotations
from html import unescape
from sqlalchemy import and_, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -23,9 +25,14 @@ from ..models import (
PostAttachment,
Source,
)
from ..utils.html_sanitize import sanitize_post_html
from ..utils.html_sanitize import (
extract_img_srcs,
rewrite_img_srcs,
sanitize_post_html,
)
from ..utils.paths import filehash_from_url
from ..utils.text import html_to_plain, truncate_at_word
from .gallery_service import thumbnail_url
from .gallery_service import image_url, thumbnail_url
from .pagination import decode_cursor, encode_cursor
DESCRIPTION_LIMIT = 280
@@ -195,11 +202,52 @@ class PostFeedService:
item["description_full"] = html_to_plain(post.description)
# Sanitized HTML body for faithful (semantic) rendering in the post view;
# detail-only (the feed list stays lightweight plain text). None when the
# post has no body.
item["description_html"] = sanitize_post_html(post.description)
# post has no body. Inline `<img>` sources are remapped to locally-served
# copies (#830 Phase 2) so the body never hotlinks the public CDN.
item["description_html"] = await self._localize_inline_images(
sanitize_post_html(post.description), post.artist_id,
)
item["external_links"] = await self._external_links_for(post.id)
return item
async def _localize_inline_images(
self, html: str | None, artist_id: int | None,
) -> str | None:
"""Rewrite a post body's inline `<img src=CDN>` to locally-served copies.
The join key is the CDN filehash the downloader persisted on each
ImageRecord (source_filehash): for every body image whose filehash maps
to a stored image of THIS artist, swap the src to /images/<path>. Images
we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
they keep hotlinking, which is the prior behavior. Scoped to the post's
artist so one creator's body never resolves to another's file."""
if not html or artist_id is None:
return html
srcs = extract_img_srcs(html)
if not srcs:
return html
# filehash -> the raw (as-in-HTML) src strings carrying it. A body can
# repeat the same image; keep every raw form so each is substituted.
by_hash: dict[str, list[str]] = {}
for raw in srcs:
fh = filehash_from_url(unescape(raw))
if fh:
by_hash.setdefault(fh, []).append(raw)
if not by_hash:
return html
rows = (await self.session.execute(
select(ImageRecord.source_filehash, ImageRecord.path)
.where(
ImageRecord.artist_id == artist_id,
ImageRecord.source_filehash.in_(list(by_hash)),
)
)).all()
replace: dict[str, str] = {}
for fh, path in rows:
for raw in by_hash.get(fh, ()):
replace[raw] = image_url(path)
return rewrite_img_srcs(html, replace)
async def _external_links_for(self, post_id: int) -> list[dict]:
"""Off-platform file-host links recorded for a post (detail-only). Each
carries its host, full url, label, and download status so the post view
+38
View File
@@ -6,6 +6,8 @@ server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used;
bleach is deprecated upstream and intentionally not used.
"""
import re
import nh3
# A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped
@@ -50,3 +52,39 @@ def sanitize_post_html(raw: str | None) -> str | None:
link_rel="noopener noreferrer",
clean_content_tags={"script", "style"},
)
# nh3 emits attributes double-quoted, so a single `src="..."` capture is enough
# to read/rewrite inline-image sources on already-sanitized markup (#830 Phase 2).
_IMG_SRC_RE = re.compile(r'(<img\b[^>]*?\bsrc=")([^"]*)(")', re.IGNORECASE)
def extract_img_srcs(html: str | None) -> list[str]:
"""The raw `src` values of every <img> in `html`, in document order, deduped
(first kept). Returned EXACTLY as they appear (still entity-escaped) so a
caller can match them back for verbatim substitution."""
if not html:
return []
out: list[str] = []
seen: set[str] = set()
for _pre, src, _post in _IMG_SRC_RE.findall(html):
if src and src not in seen:
seen.add(src)
out.append(src)
return out
def rewrite_img_srcs(html: str | None, replace: dict[str, str]) -> str | None:
"""Swap each <img src> whose raw (as-in-HTML) value is a key in `replace`
with the mapped value; others are left untouched. Used to point a post body's
inline images at locally-served copies (#830 Phase 2). Operates only on src
values of already-sanitized markup, so the result stays within the allowlist.
Returns `html` unchanged when it or `replace` is empty."""
if not html or not replace:
return html
def _sub(m: re.Match) -> str:
pre, src, post = m.group(1), m.group(2), m.group(3)
return f"{pre}{replace.get(src, src)}{post}"
return _IMG_SRC_RE.sub(_sub, html)
+19
View File
@@ -1,9 +1,28 @@
"""Filesystem path helpers — destination derivation, hash-suffixed names."""
import re
from pathlib import Path
_MAX_EXT_LEN = 16
# A Patreon/gallery CDN URL embeds a 32-char hex (MD5) path segment that is the
# file's stable per-file identity — the same role gallery-dl's `_filehash`
# plays. It is the join key between a post body `<img src=CDN>` and the local
# copy we downloaded (extract_media dedups content vs gallery images by it), so
# this ONE extractor must be used for both capture-time persistence and
# render-time matching — they cannot be allowed to drift. Match the FIRST 32-hex
# run anywhere in the URL (path or query); real CDN URLs carry exactly one.
_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
def filehash_from_url(url: str | None) -> str | None:
"""The 32-char hex (MD5) CDN identity segment of `url`, lowercased, or None
when the URL is empty / carries no such segment."""
if not url:
return None
match = _FILEHASH_RE.search(url)
return match.group(1).lower() if match else None
def safe_ext(name: str | Path) -> str:
"""Conservatively extract a short, alphanumeric file extension.
+11
View File
@@ -27,6 +27,11 @@ class SidecarData:
description: str | None
attachment_count: int | None
post_date: datetime | None
# Per-FILE CDN/origin URL the media was downloaded from (#830 Phase 2). Only
# the per-media sidecar carries it; post-only sidecars leave it None. The
# importer persists it on the ImageRecord so the body's inline `<img src>`
# can be matched (by filehash) to the local copy at render time.
source_url: str | None
raw: dict
@@ -172,6 +177,11 @@ def parse_sidecar(data: dict) -> SidecarData:
else:
post_url = _first_str(data, ("url", "post_url"))
# Per-file source URL (#830 Phase 2): the native downloader writes it into
# each media sidecar. `url` is the post permalink for Patreon (handled
# above), so source_url has its own dedicated key and never collides.
source_url = _first_str(data, ("source_url",))
return SidecarData(
platform=platform,
external_post_id=external_post_id,
@@ -180,5 +190,6 @@ def parse_sidecar(data: dict) -> SidecarData:
description=description,
attachment_count=attachment_count,
post_date=post_date,
source_url=source_url,
raw=data,
)
+42 -1
View File
@@ -1,4 +1,8 @@
from backend.app.utils.html_sanitize import sanitize_post_html
from backend.app.utils.html_sanitize import (
extract_img_srcs,
rewrite_img_srcs,
sanitize_post_html,
)
def test_none_and_blank_return_none():
@@ -52,3 +56,40 @@ def test_headings_lists_and_images_survive():
def test_image_javascript_src_dropped():
out = sanitize_post_html('<img src="javascript:alert(1)" alt="x">')
assert "javascript:" not in out
# --- inline-image localization helpers (#830 Phase 2) ----------------------
def test_extract_img_srcs_in_order_deduped():
html = (
'<p>a</p><img src="https://cdn.test/1.jpg">'
'<img alt="x" src="https://cdn.test/2.jpg" width="10">'
'<img src="https://cdn.test/1.jpg">'
)
assert extract_img_srcs(html) == [
"https://cdn.test/1.jpg",
"https://cdn.test/2.jpg",
]
def test_extract_img_srcs_empty():
assert extract_img_srcs(None) == []
assert extract_img_srcs("<p>no images</p>") == []
def test_rewrite_img_srcs_swaps_only_mapped():
html = (
'<img src="https://cdn.test/1.jpg" alt="a">'
'<img src="https://cdn.test/2.jpg">'
)
out = rewrite_img_srcs(html, {"https://cdn.test/1.jpg": "/images/local/1.jpg"})
assert 'src="/images/local/1.jpg"' in out
assert 'alt="a"' in out # other attributes preserved
assert 'src="https://cdn.test/2.jpg"' in out # unmapped left alone
def test_rewrite_img_srcs_noop_on_empty():
html = '<img src="https://cdn.test/1.jpg">'
assert rewrite_img_srcs(html, {}) == html
assert rewrite_img_srcs(None, {"a": "b"}) is None
+19
View File
@@ -3,12 +3,31 @@ from pathlib import Path
from backend.app.utils.paths import (
derive_subdir,
derive_top_level_artist,
filehash_from_url,
hash_suffixed_name,
)
IMPORT = Path("/import")
def test_filehash_from_url_extracts_lowercased_md5():
url = "https://c10.patreonusercontent.com/4/patreon-media/p/post/1/AbC123DeF456789012345678901234EF/1/img.png?token=x"
assert filehash_from_url(url) == "abc123def456789012345678901234ef"
def test_filehash_from_url_none_and_no_hash():
assert filehash_from_url(None) is None
assert filehash_from_url("") is None
assert filehash_from_url("https://example.test/a.png") is None
def test_filehash_from_url_query_unescaping_irrelevant():
# The hash sits in the path before the query, so entity-escaped ampersands
# in the query never affect the match (render-time matches escaped srcs).
base = "https://cdn.test/p/0123456789abcdef0123456789ABCDEF/x.jpg"
assert filehash_from_url(base + "?a=1&amp;b=2") == "0123456789abcdef0123456789abcdef"
def test_derive_subdir_nested():
assert derive_subdir(Path("/import/Alice/sub/x.png"), IMPORT) == "Alice/sub"
+3
View File
@@ -175,6 +175,9 @@ def test_sidecar_written_and_findable(tmp_path):
assert data["id"] == "1001"
assert data["title"] == "My Post Title"
assert data["url"] == "https://www.patreon.com/posts/1001"
# #830 Phase 2: the per-media sidecar records THIS file's CDN URL so the
# importer can persist its filehash for inline-image localization.
assert data["source_url"] == "https://cdn.patreon.com/media1.png"
def test_skip_seen(tmp_path):
+56
View File
@@ -438,6 +438,62 @@ async def test_get_post_returns_sanitized_html_body(db):
assert "<script>" not in item["description_html"]
@pytest.mark.asyncio
async def test_get_post_localizes_inline_images(db):
"""#830 Phase 2: a body <img src=CDN> whose filehash matches a stored image
of this artist is rewritten to the local /images path; an unmatched src is
left hotlinking."""
artist = await _seed_artist(db, "alice-inline")
src = await _seed_source(db, artist.id, "patreon", "https://p/alice-inline")
fh = "0123456789abcdef0123456789abcdef"
matched = f"https://cdn.test/p/{fh}/img.png"
unmatched = "https://cdn.test/p/ffffffffffffffffffffffffffffffff/other.png"
p = await _seed_post(
db, src.id, external_id="INLINE", post_date=datetime.now(UTC),
description=f'<p>x</p><img src="{matched}"><img src="{unmatched}">',
)
db.add(ImageRecord(
path="/images/alice-inline/patreon/post/01_img.png",
sha256=f"{p.id:064d}",
size_bytes=10, mime="image/png", width=10, height=10,
origin="downloaded", primary_post_id=p.id, artist_id=artist.id,
source_url=matched, source_filehash=fh,
))
await db.commit()
item = await PostFeedService(db).get_post(p.id)
html = item["description_html"]
assert 'src="/images/alice-inline/patreon/post/01_img.png"' in html
assert matched not in html # CDN src swapped out
assert unmatched in html # no local copy → still hotlinked
@pytest.mark.asyncio
async def test_get_post_inline_image_scoped_to_artist(db):
"""A matching filehash owned by a DIFFERENT artist must not leak into this
post's body — localization is artist-scoped."""
a1 = await _seed_artist(db, "owner-art")
a2 = await _seed_artist(db, "other-art")
src = await _seed_source(db, a1.id, "patreon", "https://p/owner-art")
fh = "abcabcabcabcabcabcabcabcabcabc12"
cdn = f"https://cdn.test/p/{fh}/x.png"
p = await _seed_post(
db, src.id, external_id="SCOPED", post_date=datetime.now(UTC),
description=f'<img src="{cdn}">',
)
# The only image with this filehash belongs to a DIFFERENT artist.
db.add(ImageRecord(
path="/images/other/x.png", sha256=f"{p.id:064d}",
size_bytes=10, mime="image/png", width=10, height=10,
origin="downloaded", artist_id=a2.id,
source_url=cdn, source_filehash=fh,
))
await db.commit()
item = await PostFeedService(db).get_post(p.id)
assert cdn in item["description_html"] # not localized (wrong artist)
@pytest.mark.asyncio
async def test_get_post_returns_external_links(db):
artist = await _seed_artist(db, "alice-extlinks")
+19
View File
@@ -101,6 +101,25 @@ def test_sidecar_creates_provenance(importer, import_layout):
assert rec.effective_date == post.post_date
def test_sidecar_source_url_persists_filehash(importer, import_layout):
"""#830 Phase 2: a media sidecar's source_url lands on the ImageRecord as
source_url + the lowercased CDN filehash (the inline-image join key)."""
import_root, _ = import_layout
m = import_root / "Alice" / "b.jpg"
_split(m, "v")
_sidecar(m, {
"category": "patreon", "id": 901, "title": "Body post",
"source_url": "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/b.jpg",
})
r = importer.import_one(m)
assert r.status == "imported"
rec = importer.session.get(ImageRecord, r.image_id)
assert rec.source_url == (
"https://cdn.test/p/0123456789ABCDEF0123456789abcdef/b.jpg"
)
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
def test_reimport_same_post_idempotent(importer, import_layout):
import_root, _ = import_layout
# Threshold 0: only an exact phash match collapses. Orthogonal splits
+9
View File
@@ -119,6 +119,15 @@ def test_parse_message_used_as_description_fallback():
assert sd.description == "hello channel"
def test_parse_source_url_present_and_absent():
"""The per-media sidecar carries a `source_url` (#830 Phase 2); a post-only
sidecar (no media file) omits it → None."""
sd = parse_sidecar({"category": "patreon", "id": "1",
"source_url": "https://cdn.test/abc.png"})
assert sd.source_url == "https://cdn.test/abc.png"
assert parse_sidecar({"category": "patreon", "id": "1"}).source_url is None
def test_parse_date_epoch_and_unparseable_and_naive():
assert parse_sidecar({"timestamp": 1690857602}).post_date.tzinfo is not None
assert parse_sidecar({"date": "not-a-date"}).post_date is None