M6 1901: URL capture with link-preview unfurl (SSRF-hardened)

Paste a link → fetch its OpenGraph/meta preview (title, description, image,
site) and show a rich card. User-triggered + persisted (never auto-fetches;
cached so it never re-fetches). Opt-in via a new admin setting
enable_url_unfurl (default on, rule 26).

Security (the whole point of this task): a new dependency-free unfurl.py
does the fetch with layered SSRF defenses — http/https only; resolve the
host and reject EVERY non-public address (private/loopback/link-local/
reserved/multicast/unspecified — blocks 169.254.169.254 etc.); connect to
the vetted IP with SNI so DNS-rebinding can't slip through; ≤3 redirects
each re-validated; 5s timeout; 512 KB cap; text/html only; blocking IO in a
worker thread. No server-side image fetch — the og:image URL is loaded by
the browser.

- note_link_previews table (migration 0020), one per (note, url); serialized
  inline on notes (+ rides the sync pull feed read-only).
- POST /api/notes/<id>/unfurl {url} (owner-scoped, setting-gated, 502 on
  fetch failure); DELETE /api/notes/<id>/previews/<id>.
- enable_url_unfurl exposed in public config so the UI hides the affordance
  when disabled.

Frontend: LinkPreview.vue card; editor detects URLs in the body and offers a
"Preview <domain>" chip per un-previewed link (ensureDraft first), renders
preview cards with remove; card shows previews read-only. New link icon;
notes-store unfurl()/deletePreview().

Tests (DB-free): is_public_ip range blocking, validate_url scheme/parts,
extract_preview (OG + <title> fallback + relative-image resolve), endpoint
auth-guards.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FRgehjoz7Yv8LkUfADxACm

src/thoughtsync/models/all.py

@@ -11,6 +11,7 @@ from . import (  # noqa: F401
     note_attachment,
     note_item,
     note_link,
+    note_link_preview,
     note_revision,
     settings,
     share,

src/thoughtsync/models/note_link_preview.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import DateTime, ForeignKey, Text, UniqueConstraint, func
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import Mapped, mapped_column
+
+from . import Base
+
+
+class NoteLinkPreview(Base):
+    """A cached OpenGraph/meta preview for a URL in a note, fetched server-side on the
+    user's request (see unfurl.py). Stored so it never re-fetches. One per (note, url)."""
+
+    __tablename__ = "note_link_previews"
+    __table_args__ = (UniqueConstraint("note_id", "url", name="uq_note_link_previews_note_url"),)
+
+    id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    note_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("notes.id", ondelete="CASCADE"), nullable=False
+    )
+    url: Mapped[str] = mapped_column(Text(), nullable=False)
+    title: Mapped[str | None] = mapped_column(Text(), nullable=True)
+    description: Mapped[str | None] = mapped_column(Text(), nullable=True)
+    image_url: Mapped[str | None] = mapped_column(Text(), nullable=True)
+    site_name: Mapped[str | None] = mapped_column(Text(), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())

src/thoughtsync/notes.py

@@ -18,11 +18,13 @@ from .auth import login_required
 from .config import Config
 from .db import session_scope
 from .settings import get_setting
+from .unfurl import UnfurlError, unfurl
 from .models.label import Label, NoteLabel
 from .models.note import NOTE_COLORS, Note
 from .models.note_attachment import NoteAttachment
 from .models.note_item import NoteItem
 from .models.note_link import NoteLink
+from .models.note_link_preview import NoteLinkPreview
 from .models.note_revision import NoteRevision
 
 ALLOWED_IMAGE_MIMES = {"image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif", "image/webp": ".webp"}
@@ -173,6 +175,34 @@ async def _attachments_for_notes(db, note_ids: list) -> dict:
     return result
 
 
+def _serialize_preview(p: NoteLinkPreview) -> dict:
+    return {
+        "id": str(p.id),
+        "url": p.url,
+        "title": p.title,
+        "description": p.description,
+        "image_url": p.image_url,
+        "site_name": p.site_name,
+    }
+
+
+async def _previews_for_notes(db, note_ids: list) -> dict:
+    """Map note_id -> [link previews] in one query."""
+    result: dict = {}
+    if not note_ids:
+        return result
+    rows = (
+        await db.scalars(
+            select(NoteLinkPreview)
+            .where(NoteLinkPreview.note_id.in_(note_ids))
+            .order_by(NoteLinkPreview.created_at)
+        )
+    ).all()
+    for p in rows:
+        result.setdefault(p.note_id, []).append(_serialize_preview(p))
+    return result
+
+
 async def _serialize_note(db, note: Note) -> dict:
     data = note.serialize()
     labels = await _labels_for_notes(db, [note.id])
@@ -181,6 +211,8 @@ async def _serialize_note(db, note: Note) -> dict:
     data["items"] = items.get(note.id, [])
     attachments = await _attachments_for_notes(db, [note.id])
     data["attachments"] = attachments.get(note.id, [])
+    previews = await _previews_for_notes(db, [note.id])
+    data["previews"] = previews.get(note.id, [])
     return data
 
 
@@ -189,12 +221,14 @@ async def _serialize_notes(db, notes: list) -> list:
     labels_map = await _labels_for_notes(db, ids)
     items_map = await _items_for_notes(db, ids)
     attach_map = await _attachments_for_notes(db, ids)
+    preview_map = await _previews_for_notes(db, ids)
     out = []
     for n in notes:
         data = n.serialize()
         data["labels"] = labels_map.get(n.id, [])
         data["items"] = items_map.get(n.id, [])
         data["attachments"] = attach_map.get(n.id, [])
+        data["previews"] = preview_map.get(n.id, [])
         out.append(data)
     return out
 
@@ -1320,6 +1354,67 @@ async def delete_attachment(note_id: str, att_id: str):
     return jsonify(result)
 
 
+@bp.post("/<note_id>/unfurl")
+@login_required
+async def unfurl_link(note_id: str):
+    """Fetch a link preview for a URL in this note and store it. Opt-in via the
+    enable_url_unfurl setting; SSRF-guarded server-side fetch (see unfurl.py)."""
+    data = await request.get_json(silent=True) or {}
+    url = (data.get("url") or "").strip()
+    if not url:
+        return jsonify({"error": "url is required"}), 400
+    async with session_scope() as db:
+        note = await _get_owned(db, note_id)
+        if note is None:
+            return jsonify({"error": "not found"}), 404
+        if not await get_setting(db, "enable_url_unfurl"):
+            return jsonify({"error": "link previews are disabled"}), 403
+    # Fetch OUTSIDE the DB session — network IO shouldn't hold a connection.
+    try:
+        preview = await unfurl(url)
+    except UnfurlError as e:
+        return jsonify({"error": str(e)}), 502
+    async with session_scope() as db:
+        note = await _get_owned(db, note_id)
+        if note is None:
+            return jsonify({"error": "not found"}), 404
+        # Keyed by the ORIGINAL pasted url (what the note body contains, so the client
+        # matches it) — re-unfurling the same link updates the cached preview in place.
+        row = await db.scalar(
+            select(NoteLinkPreview).where(NoteLinkPreview.note_id == note.id, NoteLinkPreview.url == url)
+        )
+        if row is None:
+            row = NoteLinkPreview(note_id=note.id, url=url)
+            db.add(row)
+        row.title = preview["title"]
+        row.description = preview["description"]
+        row.image_url = preview["image_url"]
+        row.site_name = preview["site_name"]
+        await db.commit()
+        return jsonify(await _serialize_note(db, note)), 201
+
+
+@bp.delete("/<note_id>/previews/<preview_id>")
+@login_required
+async def delete_preview(note_id: str, preview_id: str):
+    try:
+        pid = uuid.UUID(preview_id)
+    except (ValueError, TypeError):
+        return jsonify({"error": "not found"}), 404
+    async with session_scope() as db:
+        note = await _get_owned(db, note_id)
+        if note is None:
+            return jsonify({"error": "not found"}), 404
+        row = await db.scalar(
+            select(NoteLinkPreview).where(NoteLinkPreview.id == pid, NoteLinkPreview.note_id == note.id)
+        )
+        if row is None:
+            return jsonify({"error": "not found"}), 404
+        await db.delete(row)
+        await db.commit()
+        return jsonify(await _serialize_note(db, note))
+
+
 @bp.post("/<note_id>/trash")
 @login_required
 async def trash_note(note_id: str):

src/thoughtsync/settings.py

@@ -51,6 +51,15 @@ REGISTRY: list[SettingDef] = [
         "Largest single file that can be attached to a note. Capped by the server body limit.",
         "Attachments",
     ),
+    SettingDef(
+        "enable_url_unfurl",
+        "bool",
+        True,
+        "Link previews",
+        "Let the server fetch a page's title/description/image to preview pasted links. "
+        "The server contacts the linked site; private/internal addresses are always blocked.",
+        "Links",
+    ),
 ]
 
 _BY_KEY: dict[str, SettingDef] = {d.key: d for d in REGISTRY}
@@ -112,6 +121,7 @@ async def get_public_config(db) -> dict:
     return {
         "site_name": await get_setting(db, "site_name"),
         "allow_registration": await get_setting(db, "allow_registration"),
+        "enable_url_unfurl": await get_setting(db, "enable_url_unfurl"),
     }
 
 

src/thoughtsync/unfurl.py

@@ -0,0 +1,194 @@
+"""SSRF-hardened link unfurling — fetch a URL server-side and extract an OG/meta
+preview. Dependency-free (stdlib only), matching the project's hand-rolled ethos.
+
+The server fetching arbitrary user-supplied URLs is a classic SSRF surface, so the
+defenses are deliberate and layered:
+  - http/https only (no file://, gopher://, …).
+  - Resolve the host and require EVERY resolved address to be public — reject
+    private / loopback / link-local / reserved / multicast / unspecified ranges.
+  - Connect to the exact vetted IP (with SNI = the hostname), so a name that
+    re-resolves to an internal address between check and connect (DNS rebinding)
+    can't slip through.
+  - At most 3 redirects, each hop re-validated the same way.
+  - 5s timeout, 512 KB body cap, text/html only.
+No AI — this just parses OpenGraph/Twitter/`<title>` meta tags.
+"""
+from __future__ import annotations
+
+import asyncio
+import http.client
+import ipaddress
+import re
+import socket
+import ssl
+from html import unescape
+from urllib.parse import urljoin, urlparse
+
+MAX_REDIRECTS = 3
+TIMEOUT_S = 5.0
+MAX_BYTES = 512 * 1024
+USER_AGENT = "ThoughtSync-LinkPreview/1.0"
+_REDIRECT_CODES = {301, 302, 303, 307, 308}
+
+
+class UnfurlError(Exception):
+    """A URL could not be safely unfurled (bad scheme, blocked address, fetch error)."""
+
+
+def is_public_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    """True only for globally-routable addresses — everything internal is rejected."""
+    return not (
+        ip.is_private
+        or ip.is_loopback
+        or ip.is_link_local
+        or ip.is_multicast
+        or ip.is_reserved
+        or ip.is_unspecified
+    )
+
+
+def validate_url(raw: str) -> tuple[str, str, int, str]:
+    """Parse a safe http(s) URL → (scheme, host, port, path+query). Raise otherwise."""
+    parsed = urlparse((raw or "").strip())
+    if parsed.scheme not in ("http", "https"):
+        raise UnfurlError("only http and https links can be previewed")
+    host = parsed.hostname
+    if not host:
+        raise UnfurlError("that link has no host")
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    path = parsed.path or "/"
+    if parsed.query:
+        path = f"{path}?{parsed.query}"
+    return parsed.scheme, host, port, path
+
+
+def _resolve_public(host: str, port: int) -> str:
+    """Resolve host; require ALL resolved addresses to be public. Return one vetted IP."""
+    try:
+        infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
+    except socket.gaierror as e:
+        raise UnfurlError("could not resolve that host") from e
+    chosen: str | None = None
+    for info in infos:
+        addr = info[4][0]
+        try:
+            ip = ipaddress.ip_address(addr.split("%")[0])  # strip any IPv6 zone id
+        except ValueError as e:
+            raise UnfurlError("could not resolve that host") from e
+        if not is_public_ip(ip):
+            raise UnfurlError("that address isn't allowed")
+        if chosen is None:
+            chosen = addr
+    if chosen is None:
+        raise UnfurlError("could not resolve that host")
+    return chosen
+
+
+def _fetch_once(scheme: str, host: str, port: int, path: str) -> tuple[int, str | None, bytes]:
+    """One blocking GET to the VETTED public IP for `host`. Returns (status, location,
+    body). Reads at most MAX_BYTES of a text/html body."""
+    ip = _resolve_public(host, port)
+    sock: socket.socket = socket.create_connection((ip, port), timeout=TIMEOUT_S)
+    try:
+        if scheme == "https":
+            ctx = ssl.create_default_context()
+            sock = ctx.wrap_socket(sock, server_hostname=host)  # SNI + cert check vs host
+        sock.settimeout(TIMEOUT_S)  # ensure reads can't hang after the TLS wrap
+        conn = http.client.HTTPConnection(host, port, timeout=TIMEOUT_S)
+        conn.sock = sock  # use our pre-vetted (and, for https, wrapped) socket
+        conn.request(
+            "GET",
+            path,
+            headers={
+                "User-Agent": USER_AGENT,
+                "Accept": "text/html,application/xhtml+xml",
+                "Accept-Encoding": "identity",
+                "Connection": "close",
+            },
+        )
+        resp = conn.getresponse()
+        headers = {k.lower(): v for k, v in resp.getheaders()}
+        if resp.status in _REDIRECT_CODES:
+            return resp.status, headers.get("location"), b""
+        ctype = headers.get("content-type", "").split(";")[0].strip().lower()
+        if ctype and ctype not in ("text/html", "application/xhtml+xml"):
+            raise UnfurlError("that link isn't a web page")
+        return resp.status, None, resp.read(MAX_BYTES)
+    finally:
+        try:
+            sock.close()
+        except OSError:
+            pass
+
+
+def _fetch(url: str) -> tuple[str, bytes]:
+    """Follow up to MAX_REDIRECTS, re-validating each hop. Returns (final_url, html)."""
+    current = url
+    for _ in range(MAX_REDIRECTS + 1):
+        scheme, host, port, path = validate_url(current)
+        try:
+            status, location, body = _fetch_once(scheme, host, port, path)
+        except UnfurlError:
+            raise
+        except (OSError, ssl.SSLError, http.client.HTTPException) as e:
+            raise UnfurlError("could not fetch that link") from e
+        if status in _REDIRECT_CODES and location:
+            current = urljoin(current, location)
+            continue
+        if status >= 400:
+            raise UnfurlError(f"the site returned an error ({status})")
+        return current, body
+    raise UnfurlError("too many redirects")
+
+
+def _meta_content(html: str, key: str) -> str | None:
+    """The content of a <meta property|name="key" content="…"> tag (either attr order)."""
+    k = re.escape(key)
+    for pat in (
+        rf'<meta[^>]+(?:property|name)=["\']{k}["\'][^>]*content=["\']([^"\']*)["\']',
+        rf'<meta[^>]+content=["\']([^"\']*)["\'][^>]*(?:property|name)=["\']{k}["\']',
+    ):
+        m = re.search(pat, html, re.IGNORECASE | re.DOTALL)
+        if m:
+            val = unescape(m.group(1)).strip()
+            if val:
+                return val
+    return None
+
+
+def extract_preview(final_url: str, body: bytes) -> dict:
+    """Pull a link preview (title/description/image/site) from HTML. OpenGraph first,
+    then Twitter cards, then <title> / bare <meta name=description>."""
+    html = body.decode("utf-8", errors="replace")
+    title = _meta_content(html, "og:title") or _meta_content(html, "twitter:title")
+    if not title:
+        tm = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+        if tm:
+            title = unescape(re.sub(r"\s+", " ", tm.group(1)).strip()) or None
+    description = (
+        _meta_content(html, "og:description")
+        or _meta_content(html, "twitter:description")
+        or _meta_content(html, "description")
+    )
+    image = _meta_content(html, "og:image") or _meta_content(html, "twitter:image")
+    if image:
+        image = urljoin(final_url, image)  # resolve a relative image path
+        if urlparse(image).scheme not in ("http", "https"):
+            image = None
+    site_name = _meta_content(html, "og:site_name")
+    host = urlparse(final_url).hostname or ""
+    return {
+        "url": final_url,
+        "title": (title or host)[:300],
+        "description": description[:500] if description else None,
+        "image_url": image[:1000] if image else None,
+        "site_name": (site_name or host)[:100] or None,
+    }
+
+
+async def unfurl(url: str) -> dict:
+    """Fetch `url` server-side (SSRF-guarded) and return a link preview. The blocking
+    socket IO runs in a worker thread so it never stalls the event loop. Raises
+    UnfurlError on any failure."""
+    final_url, body = await asyncio.to_thread(_fetch, url)
+    return extract_preview(final_url, body)