From 96c29c370be67b343c008103c2ed3a314209954c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 14 Jun 2026 16:39:58 -0400 Subject: [PATCH] feat(ingest): localize inline post-body images to local copies (Phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render a post body faithfully by serving our stored copies of inline images instead of hotlinking the public CDN. The join key is the CDN filehash (32-hex MD5) shared between a body and the media URL we downloaded (the same identity extract_media dedups by): - utils.paths.filehash_from_url — one source of truth for the extractor; patreon_client._filehash now delegates so capture- and render-time hashing cannot drift. - ImageRecord gains source_url (provenance) + source_filehash (indexed match key); migration 0051. - the per-media sidecar carries the file's source_url; the importer persists it (NULL-only) on the ImageRecord via _apply_sidecar. - post_feed_service.get_post remaps body -> /images/ for every inline image whose filehash maps to a stored image of THIS artist; unmatched / pre-Phase-2 images keep hotlinking. Pre-existing on-disk images have no filehash yet, so they fall back to hotlinking until re-downloaded; localization is forward-looking. Co-Authored-By: Claude Opus 4.8 --- .../versions/0051_image_source_provenance.py | 38 +++++++++++++ backend/app/models/image_record.py | 12 ++++ backend/app/services/importer.py | 9 +++ backend/app/services/patreon_client.py | 15 ++--- backend/app/services/patreon_downloader.py | 25 +++++++-- backend/app/services/post_feed_service.py | 56 +++++++++++++++++-- backend/app/utils/html_sanitize.py | 38 +++++++++++++ backend/app/utils/paths.py | 19 +++++++ backend/app/utils/sidecar.py | 11 ++++ tests/test_html_sanitize.py | 43 +++++++++++++- tests/test_paths.py | 19 +++++++ tests/test_patreon_downloader.py | 3 + tests/test_post_feed_service.py | 56 +++++++++++++++++++ tests/test_sidecar_import.py | 19 +++++++ tests/test_sidecar_util.py | 9 +++ 15 files changed, 350 insertions(+), 22 deletions(-) create mode 100644 alembic/versions/0051_image_source_provenance.py diff --git a/alembic/versions/0051_image_source_provenance.py b/alembic/versions/0051_image_source_provenance.py new file mode 100644 index 0000000..595077d --- /dev/null +++ b/alembic/versions/0051_image_source_provenance.py @@ -0,0 +1,38 @@ +"""image_record: source_url + source_filehash (inline-image localization) + +#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline +images instead of hotlinking the public CDN. The join key between a body +`` and the local file is the CDN's 32-hex filehash (the same +identity extract_media dedups by). Persist it (indexed) plus the full source +URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing +rows — those fall back to hotlinking until re-downloaded. + +Revision ID: 0051 +Revises: 0050 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0051" +down_revision: Union[str, None] = "0050" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True)) + op.add_column( + "image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True) + ) + op.create_index( + "ix_image_record_source_filehash", "image_record", ["source_filehash"] + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_source_filehash", table_name="image_record") + op.drop_column("image_record", "source_filehash") + op.drop_column("image_record", "source_url") diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index ac57c9a..07cb0ec 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -49,6 +49,18 @@ class ImageRecord(Base): # Thumbnail (populated by FC-2) thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True) + # Source provenance for downloaded media (#830 Phase 2). `source_url` is the + # CDN/origin URL the file was fetched from (debugging + future re-fetch). + # `source_filehash` is the URL's 32-hex CDN identity segment + # (utils.paths.filehash_from_url) — the JOIN KEY that maps a post body's + # inline `` back to this local copy so the rendered body serves + # our stored image instead of hotlinking the public source. Indexed for the + # render-time lookup. NULL for filesystem-imported / pre-Phase-2 rows. + source_url: Mapped[str | None] = mapped_column(Text, nullable=True) + source_filehash: Mapped[str | None] = mapped_column( + String(32), nullable=True, index=True + ) + # Origin / provenance pointers origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False) primary_post_id: Mapped[int | None] = mapped_column( diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index d81ba34..c9b5abe 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -35,6 +35,7 @@ from ..utils import safe_probe from ..utils.paths import ( derive_subdir, derive_top_level_artist, + filehash_from_url, hash_suffixed_name, safe_ext, ) @@ -1046,6 +1047,14 @@ class Importer: if record.artist_id is None: record.artist_id = artist.id + # #830 Phase 2: persist the file's CDN source identity (NULL-only, so a + # re-import / enrich-on-duplicate never clobbers it) — the filehash is + # the join key that lets post_feed_service remap the body's inline + # `` to this local copy at render time. + if sd.source_url and record.source_filehash is None: + record.source_url = sd.source_url + record.source_filehash = filehash_from_url(sd.source_url) + if explicit_source is not None: src = explicit_source else: diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index af56aca..f79f627 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -39,7 +39,7 @@ from urllib.parse import parse_qs, urlsplit import requests -from ..utils.paths import safe_ext +from ..utils.paths import filehash_from_url, safe_ext log = logging.getLogger(__name__) @@ -90,12 +90,6 @@ _FIELDS_POST = ( _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name" _FIELDS_CAMPAIGN = "name,url" -# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is -# Patreon's stable per-file identity and is what we dedup + ledger against. -# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere -# in the URL (path or query); real Patreon CDN URLs carry exactly one. -_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") - # Inline post `content` is HTML; images are emitted as . # Pull every src; downstream dedup collapses any that duplicate a gallery item # by filehash. Tolerant of attribute ordering and single/double quotes. @@ -188,10 +182,9 @@ def _load_session(cookies_path: str | Path | None) -> requests.Session: def _filehash(url: str) -> str | None: - if not url: - return None - match = _FILEHASH_RE.search(url) - return match.group(1).lower() if match else None + # Delegate to the shared extractor (utils.paths) so capture-time persistence + # and render-time inline-image matching use the EXACT same identity. + return filehash_from_url(url) def _basename_from_url(url: str) -> str: diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 42540cb..1f7443d 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -294,7 +294,7 @@ class PatreonDownloader: path=quarantine_dest, error=reason, ) - self._write_sidecar(post, out_path) + self._write_sidecar(post, out_path, source_url=media.url) return MediaOutcome(media=media, status="downloaded", path=out_path, error=None) # -- download seams ---------------------------------------------------- @@ -490,7 +490,9 @@ class PatreonDownloader: # -- sidecar ----------------------------------------------------------- - def _write_sidecar(self, post: dict, media_path: Path) -> Path: + def _write_sidecar( + self, post: dict, media_path: Path, *, source_url: str | None = None + ) -> Path: """Write the importer-consumed sidecar next to `media_path`. Patreon uses base-default sidecar keys (parse_sidecar maps @@ -498,13 +500,22 @@ class PatreonDownloader: content->description, published_at->post_date, url->post_url). Patreon registers no derive_post_url, so `url` is trusted as the permalink — we pass the post's attributes.url. - """ - return self._write_sidecar_data(post, media_path.with_suffix(".json")) - def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path: + `source_url` is THIS file's CDN URL (#830 Phase 2). The importer persists + its filehash on the ImageRecord so the post body's inline `` can + be remapped to the local copy at render time. + """ + return self._write_sidecar_data( + post, media_path.with_suffix(".json"), source_url=source_url + ) + + def _write_sidecar_data( + self, post: dict, sidecar_path: Path, *, source_url: str | None = None + ) -> Path: """Serialize the post's metadata to `sidecar_path`. Shared by the per-media sidecar (next to each file) and the post-only sidecar - (`write_post_record`, for media-less posts).""" + (`write_post_record`, for media-less posts). `source_url` is set only for + the per-media sidecar — a media-less post has no source file.""" attrs = post.get("attributes") or {} title = attrs.get("title") content = attrs.get("content") @@ -530,6 +541,8 @@ class PatreonDownloader: "published_at": published if isinstance(published, str) else None, "url": url if isinstance(url, str) else None, } + if source_url: + data["source_url"] = source_url sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index ad053d4..d21f19d 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -11,6 +11,8 @@ attachments from PostAttachment) so the API layer can jsonify directly. """ from __future__ import annotations +from html import unescape + from sqlalchemy import and_, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession @@ -23,9 +25,14 @@ from ..models import ( PostAttachment, Source, ) -from ..utils.html_sanitize import sanitize_post_html +from ..utils.html_sanitize import ( + extract_img_srcs, + rewrite_img_srcs, + sanitize_post_html, +) +from ..utils.paths import filehash_from_url from ..utils.text import html_to_plain, truncate_at_word -from .gallery_service import thumbnail_url +from .gallery_service import image_url, thumbnail_url from .pagination import decode_cursor, encode_cursor DESCRIPTION_LIMIT = 280 @@ -195,11 +202,52 @@ class PostFeedService: item["description_full"] = html_to_plain(post.description) # Sanitized HTML body for faithful (semantic) rendering in the post view; # detail-only (the feed list stays lightweight plain text). None when the - # post has no body. - item["description_html"] = sanitize_post_html(post.description) + # post has no body. Inline `` sources are remapped to locally-served + # copies (#830 Phase 2) so the body never hotlinks the public CDN. + item["description_html"] = await self._localize_inline_images( + sanitize_post_html(post.description), post.artist_id, + ) item["external_links"] = await self._external_links_for(post.id) return item + async def _localize_inline_images( + self, html: str | None, artist_id: int | None, + ) -> str | None: + """Rewrite a post body's inline `` to locally-served copies. + + The join key is the CDN filehash the downloader persisted on each + ImageRecord (source_filehash): for every body image whose filehash maps + to a stored image of THIS artist, swap the src to /images/. Images + we never captured (or pre-Phase-2 rows with no filehash) are left as-is — + they keep hotlinking, which is the prior behavior. Scoped to the post's + artist so one creator's body never resolves to another's file.""" + if not html or artist_id is None: + return html + srcs = extract_img_srcs(html) + if not srcs: + return html + # filehash -> the raw (as-in-HTML) src strings carrying it. A body can + # repeat the same image; keep every raw form so each is substituted. + by_hash: dict[str, list[str]] = {} + for raw in srcs: + fh = filehash_from_url(unescape(raw)) + if fh: + by_hash.setdefault(fh, []).append(raw) + if not by_hash: + return html + rows = (await self.session.execute( + select(ImageRecord.source_filehash, ImageRecord.path) + .where( + ImageRecord.artist_id == artist_id, + ImageRecord.source_filehash.in_(list(by_hash)), + ) + )).all() + replace: dict[str, str] = {} + for fh, path in rows: + for raw in by_hash.get(fh, ()): + replace[raw] = image_url(path) + return rewrite_img_srcs(html, replace) + async def _external_links_for(self, post_id: int) -> list[dict]: """Off-platform file-host links recorded for a post (detail-only). Each carries its host, full url, label, and download status so the post view diff --git a/backend/app/utils/html_sanitize.py b/backend/app/utils/html_sanitize.py index 5748a5e..363dd77 100644 --- a/backend/app/utils/html_sanitize.py +++ b/backend/app/utils/html_sanitize.py @@ -6,6 +6,8 @@ server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used; bleach is deprecated upstream and intentionally not used. """ +import re + import nh3 # A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped @@ -50,3 +52,39 @@ def sanitize_post_html(raw: str | None) -> str | None: link_rel="noopener noreferrer", clean_content_tags={"script", "style"}, ) + + +# nh3 emits attributes double-quoted, so a single `src="..."` capture is enough +# to read/rewrite inline-image sources on already-sanitized markup (#830 Phase 2). +_IMG_SRC_RE = re.compile(r'(]*?\bsrc=")([^"]*)(")', re.IGNORECASE) + + +def extract_img_srcs(html: str | None) -> list[str]: + """The raw `src` values of every in `html`, in document order, deduped + (first kept). Returned EXACTLY as they appear (still entity-escaped) so a + caller can match them back for verbatim substitution.""" + if not html: + return [] + out: list[str] = [] + seen: set[str] = set() + for _pre, src, _post in _IMG_SRC_RE.findall(html): + if src and src not in seen: + seen.add(src) + out.append(src) + return out + + +def rewrite_img_srcs(html: str | None, replace: dict[str, str]) -> str | None: + """Swap each whose raw (as-in-HTML) value is a key in `replace` + with the mapped value; others are left untouched. Used to point a post body's + inline images at locally-served copies (#830 Phase 2). Operates only on src + values of already-sanitized markup, so the result stays within the allowlist. + Returns `html` unchanged when it or `replace` is empty.""" + if not html or not replace: + return html + + def _sub(m: re.Match) -> str: + pre, src, post = m.group(1), m.group(2), m.group(3) + return f"{pre}{replace.get(src, src)}{post}" + + return _IMG_SRC_RE.sub(_sub, html) diff --git a/backend/app/utils/paths.py b/backend/app/utils/paths.py index d9bc48e..7130e90 100644 --- a/backend/app/utils/paths.py +++ b/backend/app/utils/paths.py @@ -1,9 +1,28 @@ """Filesystem path helpers — destination derivation, hash-suffixed names.""" +import re from pathlib import Path _MAX_EXT_LEN = 16 +# A Patreon/gallery CDN URL embeds a 32-char hex (MD5) path segment that is the +# file's stable per-file identity — the same role gallery-dl's `_filehash` +# plays. It is the join key between a post body `` and the local +# copy we downloaded (extract_media dedups content vs gallery images by it), so +# this ONE extractor must be used for both capture-time persistence and +# render-time matching — they cannot be allowed to drift. Match the FIRST 32-hex +# run anywhere in the URL (path or query); real CDN URLs carry exactly one. +_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") + + +def filehash_from_url(url: str | None) -> str | None: + """The 32-char hex (MD5) CDN identity segment of `url`, lowercased, or None + when the URL is empty / carries no such segment.""" + if not url: + return None + match = _FILEHASH_RE.search(url) + return match.group(1).lower() if match else None + def safe_ext(name: str | Path) -> str: """Conservatively extract a short, alphanumeric file extension. diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py index 0d81930..1e2a7d5 100644 --- a/backend/app/utils/sidecar.py +++ b/backend/app/utils/sidecar.py @@ -27,6 +27,11 @@ class SidecarData: description: str | None attachment_count: int | None post_date: datetime | None + # Per-FILE CDN/origin URL the media was downloaded from (#830 Phase 2). Only + # the per-media sidecar carries it; post-only sidecars leave it None. The + # importer persists it on the ImageRecord so the body's inline `` + # can be matched (by filehash) to the local copy at render time. + source_url: str | None raw: dict @@ -172,6 +177,11 @@ def parse_sidecar(data: dict) -> SidecarData: else: post_url = _first_str(data, ("url", "post_url")) + # Per-file source URL (#830 Phase 2): the native downloader writes it into + # each media sidecar. `url` is the post permalink for Patreon (handled + # above), so source_url has its own dedicated key and never collides. + source_url = _first_str(data, ("source_url",)) + return SidecarData( platform=platform, external_post_id=external_post_id, @@ -180,5 +190,6 @@ def parse_sidecar(data: dict) -> SidecarData: description=description, attachment_count=attachment_count, post_date=post_date, + source_url=source_url, raw=data, ) diff --git a/tests/test_html_sanitize.py b/tests/test_html_sanitize.py index 63eeeee..5d92dae 100644 --- a/tests/test_html_sanitize.py +++ b/tests/test_html_sanitize.py @@ -1,4 +1,8 @@ -from backend.app.utils.html_sanitize import sanitize_post_html +from backend.app.utils.html_sanitize import ( + extract_img_srcs, + rewrite_img_srcs, + sanitize_post_html, +) def test_none_and_blank_return_none(): @@ -52,3 +56,40 @@ def test_headings_lists_and_images_survive(): def test_image_javascript_src_dropped(): out = sanitize_post_html('x') assert "javascript:" not in out + + +# --- inline-image localization helpers (#830 Phase 2) ---------------------- + + +def test_extract_img_srcs_in_order_deduped(): + html = ( + '

a

' + 'x' + '' + ) + assert extract_img_srcs(html) == [ + "https://cdn.test/1.jpg", + "https://cdn.test/2.jpg", + ] + + +def test_extract_img_srcs_empty(): + assert extract_img_srcs(None) == [] + assert extract_img_srcs("

no images

") == [] + + +def test_rewrite_img_srcs_swaps_only_mapped(): + html = ( + 'a' + '' + ) + out = rewrite_img_srcs(html, {"https://cdn.test/1.jpg": "/images/local/1.jpg"}) + assert 'src="/images/local/1.jpg"' in out + assert 'alt="a"' in out # other attributes preserved + assert 'src="https://cdn.test/2.jpg"' in out # unmapped left alone + + +def test_rewrite_img_srcs_noop_on_empty(): + html = '' + assert rewrite_img_srcs(html, {}) == html + assert rewrite_img_srcs(None, {"a": "b"}) is None diff --git a/tests/test_paths.py b/tests/test_paths.py index 2976d1e..4b52f97 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -3,12 +3,31 @@ from pathlib import Path from backend.app.utils.paths import ( derive_subdir, derive_top_level_artist, + filehash_from_url, hash_suffixed_name, ) IMPORT = Path("/import") +def test_filehash_from_url_extracts_lowercased_md5(): + url = "https://c10.patreonusercontent.com/4/patreon-media/p/post/1/AbC123DeF456789012345678901234EF/1/img.png?token=x" + assert filehash_from_url(url) == "abc123def456789012345678901234ef" + + +def test_filehash_from_url_none_and_no_hash(): + assert filehash_from_url(None) is None + assert filehash_from_url("") is None + assert filehash_from_url("https://example.test/a.png") is None + + +def test_filehash_from_url_query_unescaping_irrelevant(): + # The hash sits in the path before the query, so entity-escaped ampersands + # in the query never affect the match (render-time matches escaped srcs). + base = "https://cdn.test/p/0123456789abcdef0123456789ABCDEF/x.jpg" + assert filehash_from_url(base + "?a=1&b=2") == "0123456789abcdef0123456789abcdef" + + def test_derive_subdir_nested(): assert derive_subdir(Path("/import/Alice/sub/x.png"), IMPORT) == "Alice/sub" diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 3a1d15f..06d4700 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -175,6 +175,9 @@ def test_sidecar_written_and_findable(tmp_path): assert data["id"] == "1001" assert data["title"] == "My Post Title" assert data["url"] == "https://www.patreon.com/posts/1001" + # #830 Phase 2: the per-media sidecar records THIS file's CDN URL so the + # importer can persist its filehash for inline-image localization. + assert data["source_url"] == "https://cdn.patreon.com/media1.png" def test_skip_seen(tmp_path): diff --git a/tests/test_post_feed_service.py b/tests/test_post_feed_service.py index 7374743..78a01da 100644 --- a/tests/test_post_feed_service.py +++ b/tests/test_post_feed_service.py @@ -438,6 +438,62 @@ async def test_get_post_returns_sanitized_html_body(db): assert "