feat(posts): extract + record external file-host links (Phase 3)

Capture off-platform links (mega/gdrive/mediafire/dropbox/pixeldrain) embedded
in post bodies so they're never silently dropped, and surface them in the post
view. The download worker (Phase 4) walks these rows.

- link_extract.py: pure extractor — <a href> + bare URLs, unwraps Patreon
  redirect shims, PRESERVES the full url incl. #fragment (mega's key), dedups.
  Reusable by every platform (runs off Post.description).
- external_link model + migration 0049: post_id/artist_id/host/url/label/status
  /attempts/last_error/attachment_id/timing; CHECK whitelists (full enum incl.
  worker statuses up front) + (post_id,url) unique.
- importer._sync_external_links: insert-missing on both import paths
  (_apply_sidecar + upsert_post_record) so a re-import never resets a link's
  status; runs for all platforms.
- post_feed_service.get_post: returns external_links (detail-only).
- PostCard: renders the links (host chip + label + status) once expanded.
- tests: extractor (5 hosts, fragment, shim unwrap, dedup), importer (record +
  no-dup on reimport), serializer.

Refs FC #830.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/models/__init__.py

@@ -7,6 +7,7 @@ from .backup_run import BackupRun
 from .base import Base
 from .credential import Credential
 from .download_event import DownloadEvent
+from .external_link import ExternalLink
 from .image_prediction import ImagePrediction
 from .image_provenance import ImageProvenance
 from .image_record import ImageRecord
@@ -52,6 +53,7 @@ __all__ = [
     "TagKind",
     "image_tag",
     "DownloadEvent",
+    "ExternalLink",
     "ImportBatch",
     "ImportTask",
     "ImportSettings",

backend/app/models/external_link.py

@@ -0,0 +1,73 @@
+"""ExternalLink — an off-platform file-host link found in a post body.
+
+Creators host the actual files (films, packs) on mega.nz / Google Drive /
+MediaFire / Dropbox / Pixeldrain and drop the link in the post text. This row
+is the record that the link existed (so nothing is silently dropped), the
+dedup + dead-letter ledger for fetching it, and the driver the download worker
+walks. `url` keeps the FULL link including the `#fragment` (mega's decryption
+key) — truncating it makes the file undownloadable.
+
+status lifecycle: pending → downloading → downloaded | failed | dead
+(too many attempts) | skipped (host disabled). `attachment_id` links the
+captured file once a download lands (SET NULL so deleting the attachment
+doesn't delete the link record).
+"""
+
+from datetime import datetime
+
+from sqlalchemy import (
+    DateTime,
+    Float,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+    func,
+    text,
+)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+# Kept in sync with link_extract.SUPPORTED_HOSTS and the CHECK in migration 0049.
+HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
+STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead")
+
+
+class ExternalLink(Base):
+    __tablename__ = "external_link"
+    __table_args__ = (
+        # One row per (post, url). The full url (incl. #fragment) is the identity
+        # — the same file linked twice in a post collapses to one row.
+        Index("uq_external_link_post_url", "post_id", "url", unique=True),
+        Index("ix_external_link_status", "status"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    post_id: Mapped[int] = mapped_column(
+        ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    artist_id: Mapped[int | None] = mapped_column(
+        ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
+    )
+    host: Mapped[str] = mapped_column(String(16), nullable=False)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    label: Mapped[str | None] = mapped_column(Text, nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(16), nullable=False, server_default="pending"
+    )
+    attempts: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    attachment_id: Mapped[int | None] = mapped_column(
+        ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    completed_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
+    duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)

backend/app/services/importer.py

@@ -23,6 +23,7 @@ from sqlalchemy.orm import Session
 
 from ..models import (
     Artist,
+    ExternalLink,
     ImageProvenance,
     ImageRecord,
     ImportSettings,
@@ -39,6 +40,7 @@ from ..utils.paths import (
 )
 from ..utils.phash import compute_phash, find_similar
 from ..utils.sidecar import find_sidecar, parse_sidecar
+from .link_extract import extract_external_links
 from ..utils.slug import slugify
 from .archive_extractor import extract_archive, is_archive
 from .attachment_store import AttachmentStore
@@ -305,6 +307,32 @@ class Importer:
             ),
         )
 
+    def _sync_external_links(self, post: Post) -> None:
+        """Record off-platform file-host links (mega/gdrive/mediafire/dropbox/
+        pixeldrain) found in the post body, so they're never silently dropped
+        and the download worker can fetch them later. Shared by every platform's
+        import (runs off Post.description). INSERT-MISSING only — an existing row
+        keeps its status/attempts (a re-import must not reset a link already
+        downloaded or dead-lettered). Identity is (post_id, url); the full url
+        incl. #fragment is preserved by the extractor."""
+        links = extract_external_links(post.description)
+        if not links:
+            return
+        self.session.flush()  # ensure post.id is assigned before we reference it
+        existing = set(self.session.execute(
+            select(ExternalLink.url).where(ExternalLink.post_id == post.id)
+        ).scalars().all())
+        for link in links:
+            if link.url in existing:
+                continue
+            self.session.add(ExternalLink(
+                post_id=post.id,
+                artist_id=post.artist_id,
+                host=link.host,
+                url=link.url,
+                label=link.label,
+            ))
+
     def import_one(self, source: Path) -> ImportResult:
         """Dispatch by kind. Media → normal pipeline. Archive → extract
         media members (one Post via the archive-adjacent sidecar) and
@@ -767,6 +795,7 @@ class Importer:
         if sd.attachment_count is not None:
             post.attachment_count = sd.attachment_count
         post.raw_metadata = sd.raw
+        self._sync_external_links(post)
         self.session.commit()
         return True
 
@@ -1042,6 +1071,7 @@ class Importer:
         if sd.attachment_count is not None:
             post.attachment_count = sd.attachment_count
         post.raw_metadata = sd.raw
+        self._sync_external_links(post)
 
         # Race-safe (image_record_id, post_id) upsert — mirrors the
         # _find_or_create_source/post savepoint pattern. The plain

backend/app/services/link_extract.py

@@ -0,0 +1,105 @@
+"""Extract off-platform file-host links from a post body.
+
+Pure (no I/O) so it's unit-testable and reusable by EVERY in-house downloader's
+import path — every platform stores its body in `Post.description`, so running
+this there covers them all. Finds links to the supported external file hosts in
+a post's HTML body (both `<a href="...">` anchors and bare URLs in text),
+unwraps a Patreon outbound-redirect wrapper, and preserves the FULL url
+including the `#fragment` (mega.nz puts the decryption key there) and the query
+string — without those a mega download is impossible.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from html import unescape
+from urllib.parse import parse_qs, urlsplit
+
+# Supported file-host enum values (kept in sync with the external_link CHECK).
+SUPPORTED_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
+
+# Bare-domain suffix → canonical host label. Matched against the URL netloc
+# (exact or as a dotted suffix, so www./dl. subdomains resolve too).
+_HOST_MAP = {
+    "mega.nz": "mega",
+    "mega.co.nz": "mega",
+    "drive.google.com": "gdrive",
+    "mediafire.com": "mediafire",
+    "dropbox.com": "dropbox",
+    "pixeldrain.com": "pixeldrain",
+}
+
+# `<a href="...">label</a>` — DOTALL so a label spanning tags/newlines is caught.
+_HREF_RE = re.compile(
+    r"""<a\b[^>]*?\bhref=["']([^"']+)["'][^>]*>(.*?)</a>""",
+    re.IGNORECASE | re.DOTALL,
+)
+_TAG_RE = re.compile(r"<[^>]+>")
+# Bare http(s) URL in text. Stops at whitespace, quotes, angle brackets, and
+# closing brackets — but KEEPS `#`, `&`, `?`, `=` so fragments/queries survive.
+_URL_RE = re.compile(r"""https?://[^\s"'<>)\]}]+""", re.IGNORECASE)
+
+
+@dataclass(frozen=True)
+class ExtractedLink:
+    host: str            # one of SUPPORTED_HOSTS
+    url: str             # full url incl. query + #fragment
+    label: str | None    # visible anchor text, when present
+
+
+def _netloc(url: str) -> str:
+    # Lowercase host without credentials or port.
+    return urlsplit(url).netloc.lower().split("@")[-1].split(":")[0]
+
+
+def host_for(url: str) -> str | None:
+    """Canonical host label for a url, or None if it's not a supported host."""
+    netloc = _netloc(url)
+    for suffix, host in _HOST_MAP.items():
+        if netloc == suffix or netloc.endswith("." + suffix):
+            return host
+    return None
+
+
+def _unwrap(url: str) -> str:
+    """Unwrap a Patreon outbound-redirect wrapper to its inner target, so a
+    wrapped mega/gdrive link resolves to the real host. Patreon has used both
+    `www.patreon.com/...?url=<encoded>` and the `l.patreon.com` shim; check the
+    common target-param names. Non-Patreon urls pass through untouched."""
+    netloc = _netloc(url)
+    if not (netloc == "patreon.com" or netloc.endswith(".patreon.com")):
+        return url
+    qs = parse_qs(urlsplit(url).query)
+    for key in ("url", "u", "ext_url", "redirect", "target"):
+        vals = qs.get(key)
+        if vals and vals[0]:
+            return unescape(vals[0]).strip()
+    return url
+
+
+def extract_external_links(html: str | None) -> list[ExtractedLink]:
+    """All supported-host links in `html`, de-duplicated by url (first wins, so
+    an anchor's label is kept over a later bare sighting of the same url)."""
+    if not html:
+        return []
+    found: dict[str, ExtractedLink] = {}
+
+    # 1) Anchors first — they carry a human label ("Mega - Streamable").
+    for raw_href, inner in _HREF_RE.findall(html):
+        url = _unwrap(unescape(raw_href).strip())
+        host = host_for(url)
+        if host is None:
+            continue
+        label = unescape(_TAG_RE.sub("", inner)).strip() or None
+        found.setdefault(url, ExtractedLink(host=host, url=url, label=label))
+
+    # 2) Bare URLs pasted in text (no label). Trailing prose punctuation is
+    #    trimmed; the href values already captured above de-dup away here.
+    for raw in _URL_RE.findall(html):
+        url = _unwrap(unescape(raw).strip().rstrip(".,;"))
+        host = host_for(url)
+        if host is None:
+            continue
+        found.setdefault(url, ExtractedLink(host=host, url=url, label=None))
+
+    return list(found.values())

backend/app/services/post_feed_service.py

@@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..models import (
     Artist,
+    ExternalLink,
     ImageProvenance,
     ImageRecord,
     Post,
@@ -196,8 +197,26 @@ class PostFeedService:
         # detail-only (the feed list stays lightweight plain text). None when the
         # post has no body.
         item["description_html"] = sanitize_post_html(post.description)
+        item["external_links"] = await self._external_links_for(post.id)
         return item
 
+    async def _external_links_for(self, post_id: int) -> list[dict]:
+        """Off-platform file-host links recorded for a post (detail-only). Each
+        carries its host, full url, label, and download status so the post view
+        can surface them (and, later, a retry/download affordance)."""
+        rows = (await self.session.execute(
+            select(ExternalLink)
+            .where(ExternalLink.post_id == post_id)
+            .order_by(ExternalLink.id.asc())
+        )).scalars().all()
+        return [
+            {
+                "id": e.id, "host": e.host, "url": e.url,
+                "label": e.label, "status": e.status,
+            }
+            for e in rows
+        ]
+
     # --- composition helpers ---------------------------------------------
 
     async def _thumbnails_for(