From 69bf04e9484e6f64feba5cad0678c8aa062618dc Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 23 Jul 2026 08:05:24 -0400 Subject: [PATCH] M6 1901: URL capture with link-preview unfurl (SSRF-hardened) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paste a link → fetch its OpenGraph/meta preview (title, description, image, site) and show a rich card. User-triggered + persisted (never auto-fetches; cached so it never re-fetches). Opt-in via a new admin setting enable_url_unfurl (default on, rule 26). Security (the whole point of this task): a new dependency-free unfurl.py does the fetch with layered SSRF defenses — http/https only; resolve the host and reject EVERY non-public address (private/loopback/link-local/ reserved/multicast/unspecified — blocks 169.254.169.254 etc.); connect to the vetted IP with SNI so DNS-rebinding can't slip through; ≤3 redirects each re-validated; 5s timeout; 512 KB cap; text/html only; blocking IO in a worker thread. No server-side image fetch — the og:image URL is loaded by the browser. - note_link_previews table (migration 0020), one per (note, url); serialized inline on notes (+ rides the sync pull feed read-only). - POST /api/notes//unfurl {url} (owner-scoped, setting-gated, 502 on fetch failure); DELETE /api/notes//previews/. - enable_url_unfurl exposed in public config so the UI hides the affordance when disabled. Frontend: LinkPreview.vue card; editor detects URLs in the body and offers a "Preview " chip per un-previewed link (ensureDraft first), renders preview cards with remove; card shows previews read-only. New link icon; notes-store unfurl()/deletePreview(). Tests (DB-free): is_public_ip range blocking, validate_url scheme/parts, extract_preview (OG + fallback + relative-image resolve), endpoint auth-guards. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FRgehjoz7Yv8LkUfADxACm --- alembic/versions/0020_note_link_previews.py | 38 ++++ frontend/src/components/Icon.vue | 1 + frontend/src/components/LinkPreview.vue | 46 +++++ frontend/src/components/NoteCard.vue | 5 + frontend/src/components/NoteEditor.vue | 68 +++++++ frontend/src/stores/config.ts | 5 +- frontend/src/stores/notes.ts | 21 +++ src/thoughtsync/models/all.py | 1 + src/thoughtsync/models/note_link_preview.py | 29 +++ src/thoughtsync/notes.py | 95 ++++++++++ src/thoughtsync/settings.py | 10 + src/thoughtsync/unfurl.py | 194 ++++++++++++++++++++ tests/test_unfurl.py | 74 ++++++++ 13 files changed, 586 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/0020_note_link_previews.py create mode 100644 frontend/src/components/LinkPreview.vue create mode 100644 src/thoughtsync/models/note_link_preview.py create mode 100644 src/thoughtsync/unfurl.py create mode 100644 tests/test_unfurl.py diff --git a/alembic/versions/0020_note_link_previews.py b/alembic/versions/0020_note_link_previews.py new file mode 100644 index 0000000..d31abb0 --- /dev/null +++ b/alembic/versions/0020_note_link_previews.py @@ -0,0 +1,38 @@ +"""note_link_previews (URL unfurl — M6 1901) + +Revision ID: 0020 +Revises: 0019 +Create Date: 2026-07-23 + +Cached OG/meta previews for URLs in a note, fetched server-side on request +(unfurl.py) and stored so they never re-fetch. One per (note, url). +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +revision = "0020" +down_revision = "0019" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "note_link_previews", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("note_id", UUID(as_uuid=True), sa.ForeignKey("notes.id", ondelete="CASCADE"), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("title", sa.Text(), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("image_url", sa.Text(), nullable=True), + sa.Column("site_name", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.UniqueConstraint("note_id", "url", name="uq_note_link_previews_note_url"), + ) + op.create_index("ix_note_link_previews_note", "note_link_previews", ["note_id"]) + + +def downgrade() -> None: + op.drop_index("ix_note_link_previews_note", table_name="note_link_previews") + op.drop_table("note_link_previews") diff --git a/frontend/src/components/Icon.vue b/frontend/src/components/Icon.vue index 1121e8f..0871e08 100644 --- a/frontend/src/components/Icon.vue +++ b/frontend/src/components/Icon.vue @@ -27,6 +27,7 @@ const paths: Record<string, string> = { upload: '<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" x2="12" y1="3" y2="15"/>', device: '<rect width="14" height="20" x="5" y="2" rx="2" ry="2"/><path d="M12 18h.01"/>', paperclip: '<path d="m21.44 11.05-9.19 9.19a6 6 0 0 1-8.49-8.49l8.57-8.57A4 4 0 1 1 18 8.84l-8.59 8.57a2 2 0 0 1-2.83-2.83l8.49-8.48"/>', + link: '<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/>', copy: '<rect width="14" height="14" x="8" y="8" rx="2" ry="2"/><path d="M4 16c-1.1 0-2-.9-2-2V4c0-1.1.9-2 2-2h10c1.1 0 2 .9 2 2"/>', }; </script> diff --git a/frontend/src/components/LinkPreview.vue b/frontend/src/components/LinkPreview.vue new file mode 100644 index 0000000..5d60f9f --- /dev/null +++ b/frontend/src/components/LinkPreview.vue @@ -0,0 +1,46 @@ +<script setup lang="ts"> +import type { LinkPreview } from "../stores/notes"; + +defineProps<{ preview: LinkPreview; removable?: boolean }>(); +defineEmits<{ (e: "remove"): void }>(); +</script> + +<template> + <div class="group/lp relative overflow-hidden rounded-lg border border-neutral-200 dark:border-neutral-700"> + <a + :href="preview.url" + target="_blank" + rel="noopener noreferrer" + class="flex items-stretch hover:bg-neutral-50 dark:hover:bg-neutral-800/60" + > + <img + v-if="preview.image_url" + :src="preview.image_url" + alt="" + loading="lazy" + decoding="async" + class="h-auto w-24 shrink-0 self-stretch object-cover" + /> + <div class="min-w-0 flex-1 px-3 py-2"> + <p v-if="preview.site_name" class="truncate text-[11px] uppercase tracking-wide text-neutral-400"> + {{ preview.site_name }} + </p> + <p class="truncate text-sm font-medium text-neutral-800 dark:text-neutral-100"> + {{ preview.title || preview.url }} + </p> + <p v-if="preview.description" class="mt-0.5 line-clamp-2 text-xs text-neutral-500 dark:text-neutral-400"> + {{ preview.description }} + </p> + </div> + </a> + <button + v-if="removable" + type="button" + class="absolute right-1 top-1 rounded-full bg-black/50 px-1.5 text-white opacity-0 transition group-hover/lp:opacity-100" + aria-label="Remove preview" + @click="$emit('remove')" + > + × + </button> + </div> +</template> diff --git a/frontend/src/components/NoteCard.vue b/frontend/src/components/NoteCard.vue index f8669db..1621bf7 100644 --- a/frontend/src/components/NoteCard.vue +++ b/frontend/src/components/NoteCard.vue @@ -11,6 +11,7 @@ import { } from "../notes/colors"; import type { Note } from "../stores/notes"; import Icon from "./Icon.vue"; +import LinkPreview from "./LinkPreview.vue"; import MarkdownText from "./MarkdownText.vue"; import NoteChecklist from "./NoteChecklist.vue"; import { formatReminder, isOverdue } from "../notes/datetime"; @@ -162,6 +163,10 @@ onBeforeUnmount(() => document.removeEventListener("mousedown", onDocMousedown)) </span> </div> + <div v-if="note.previews.length" class="mb-2 flex flex-col gap-2"> + <LinkPreview v-for="p in note.previews" :key="p.id" :preview="p" /> + </div> + <!-- Checklist notes can't nest interactive controls in a <button>, so use a focusable div; text notes keep a semantic button. --> <template v-if="note.kind === 'list'"> diff --git a/frontend/src/components/NoteEditor.vue b/frontend/src/components/NoteEditor.vue index 96ea5d4..a530a9d 100644 --- a/frontend/src/components/NoteEditor.vue +++ b/frontend/src/components/NoteEditor.vue @@ -2,10 +2,12 @@ import { computed, nextTick, onBeforeUnmount, onMounted, ref, watch } from "vue"; import { api } from "../api/client"; import { useNotesStore } from "../stores/notes"; +import { useConfigStore } from "../stores/config"; import { useTitlesStore, type TitleEntry } from "../stores/titles"; import ColorPicker from "./ColorPicker.vue"; import Icon from "./Icon.vue"; import LabelPicker from "./LabelPicker.vue"; +import LinkPreview from "./LinkPreview.vue"; import NoteChecklist from "./NoteChecklist.vue"; import { fromLocalInput, toLocalInput } from "../notes/datetime"; import type { Note, NoteLabel, NoteRevision } from "../stores/notes"; @@ -23,6 +25,7 @@ const props = withDefaults(defineProps<{ note?: Note | null; inline?: boolean; a }); const emit = defineEmits<{ (e: "close"): void; (e: "navigate", id: string): void }>(); const notes = useNotesStore(); +const config = useConfigStore(); const titles = useTitlesStore(); const noteId = ref<string | null>(props.note?.id ?? null); @@ -68,6 +71,7 @@ const draftNote = computed<Note>(() => ({ labels: labelList.value, items: [], attachments: [], + previews: [], created_at: null, updated_at: null, })); @@ -447,6 +451,42 @@ async function uploadFile(file: File) { uploadError.value = (e as { error?: string }).error ?? "Could not upload file."; } } + +// ---- link previews (URL unfurl) ---- +const unfurling = ref<string | null>(null); // the URL currently being fetched +const unfurlError = ref(""); +// Bare http(s) URLs in the body; trailing sentence punctuation trimmed. +const URL_RE = /(https?:\/\/[^\s<>"'\])]+)/g; +const detectedUrls = computed(() => { + const out: string[] = []; + for (const m of body.value.matchAll(URL_RE)) { + const u = m[1].replace(/[.,;:!?]+$/, ""); + if (!out.includes(u)) out.push(u); + } + return out; +}); +const previewedUrls = computed(() => new Set(liveNote.value.previews.map((p) => p.url))); +const unpreviewedUrls = computed(() => detectedUrls.value.filter((u) => !previewedUrls.value.has(u))); +async function addPreview(url: string) { + const id = await ensureDraft(); + if (!id) return; + unfurling.value = url; + unfurlError.value = ""; + try { + await notes.unfurl(id, url); + } catch (e) { + unfurlError.value = (e as { error?: string }).error ?? "Couldn't fetch a preview for that link."; + } finally { + unfurling.value = null; + } +} +function shortUrl(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, ""); + } catch { + return url; + } +} async function onFileChange(e: Event) { const input = e.target as HTMLInputElement; const file = input.files?.[0]; @@ -604,6 +644,34 @@ defineExpose({ open }); </div> <p v-if="uploadError" class="text-xs text-red-600 dark:text-red-400">{{ uploadError }}</p> + <!-- Link previews: stored preview cards + one "Preview <domain>" per detected URL --> + <div v-if="liveNote.previews.length" class="flex flex-col gap-2"> + <LinkPreview + v-for="p in liveNote.previews" + :key="p.id" + :preview="p" + :removable="!liveNote.trashed" + @remove="notes.deletePreview(liveNote.id, p.id)" + /> + </div> + <div + v-if="config.enableUrlUnfurl && !liveNote.trashed && unpreviewedUrls.length" + class="flex flex-wrap gap-1.5" + > + <button + v-for="u in unpreviewedUrls" + :key="u" + type="button" + class="inline-flex items-center gap-1 rounded-full border border-neutral-200 px-2 py-0.5 text-xs text-neutral-500 hover:bg-neutral-50 focus:outline-none focus-visible:ring-2 focus-visible:ring-brand disabled:opacity-60 dark:border-neutral-700 dark:hover:bg-neutral-800" + :disabled="unfurling === u" + @click="addPreview(u)" + > + <Icon name="link" /> + {{ unfurling === u ? "Fetching…" : `Preview ${shortUrl(u)}` }} + </button> + </div> + <p v-if="unfurlError" class="text-xs text-red-600 dark:text-red-400">{{ unfurlError }}</p> + <input v-model="title" type="text" diff --git a/frontend/src/stores/config.ts b/frontend/src/stores/config.ts index 210a300..9f93358 100644 --- a/frontend/src/stores/config.ts +++ b/frontend/src/stores/config.ts @@ -6,6 +6,7 @@ interface PublicConfig { site_name: string; allow_registration: boolean; version: string; + enable_url_unfurl: boolean; } // Public, unauthenticated app config (site name, whether signups are open). @@ -13,6 +14,7 @@ export const useConfigStore = defineStore("config", () => { const siteName = ref("ThoughtSync"); const allowRegistration = ref(true); const version = ref(""); + const enableUrlUnfurl = ref(true); const loaded = ref(false); async function load(): Promise<void> { @@ -22,6 +24,7 @@ export const useConfigStore = defineStore("config", () => { siteName.value = cfg.site_name; allowRegistration.value = cfg.allow_registration; version.value = cfg.version; + enableUrlUnfurl.value = cfg.enable_url_unfurl ?? true; } catch { // Keep defaults if the config endpoint is unreachable. } finally { @@ -34,5 +37,5 @@ export const useConfigStore = defineStore("config", () => { await load(); } - return { siteName, allowRegistration, version, loaded, load, reload }; + return { siteName, allowRegistration, version, enableUrlUnfurl, loaded, load, reload }; }); diff --git a/frontend/src/stores/notes.ts b/frontend/src/stores/notes.ts index 8405dfa..3d5971b 100644 --- a/frontend/src/stores/notes.ts +++ b/frontend/src/stores/notes.ts @@ -32,6 +32,16 @@ export interface Attachment { sha256?: string | null; } +// A cached OpenGraph/meta preview for a URL in the note (server-fetched, SSRF-guarded). +export interface LinkPreview { + id: string; + url: string; + title: string | null; + description: string | null; + image_url: string | null; + site_name: string | null; +} + // A past version of a note's title+body (version history). export interface NoteRevision { id: string; @@ -57,6 +67,7 @@ export interface Note { labels: NoteLabel[]; items: ChecklistItem[]; attachments: Attachment[]; + previews: LinkPreview[]; created_at: string | null; updated_at: string | null; } @@ -174,6 +185,14 @@ export const useNotesStore = defineStore("notes", () => { reconcile(await api.del<Note>(`/api/notes/${id}/attachments/${attId}`)); } + async function unfurl(id: string, url: string): Promise<void> { + reconcile(await api.post<Note>(`/api/notes/${id}/unfurl`, { url })); + } + + async function deletePreview(id: string, previewId: string): Promise<void> { + reconcile(await api.del<Note>(`/api/notes/${id}/previews/${previewId}`)); + } + async function importNotes(file: File): Promise<{ source: string; imported: number; skipped: number }> { const form = new FormData(); form.append("file", file); @@ -262,6 +281,8 @@ export const useNotesStore = defineStore("notes", () => { deleteItem, uploadAttachment, deleteAttachment, + unfurl, + deletePreview, importNotes, fetchOne, createTitled, diff --git a/src/thoughtsync/models/all.py b/src/thoughtsync/models/all.py index 058e180..9c2f03a 100644 --- a/src/thoughtsync/models/all.py +++ b/src/thoughtsync/models/all.py @@ -11,6 +11,7 @@ from . import ( # noqa: F401 note_attachment, note_item, note_link, + note_link_preview, note_revision, settings, share, diff --git a/src/thoughtsync/models/note_link_preview.py b/src/thoughtsync/models/note_link_preview.py new file mode 100644 index 0000000..a5e18fc --- /dev/null +++ b/src/thoughtsync/models/note_link_preview.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, Text, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column + +from . import Base + + +class NoteLinkPreview(Base): + """A cached OpenGraph/meta preview for a URL in a note, fetched server-side on the + user's request (see unfurl.py). Stored so it never re-fetches. One per (note, url).""" + + __tablename__ = "note_link_previews" + __table_args__ = (UniqueConstraint("note_id", "url", name="uq_note_link_previews_note_url"),) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + note_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("notes.id", ondelete="CASCADE"), nullable=False + ) + url: Mapped[str] = mapped_column(Text(), nullable=False) + title: Mapped[str | None] = mapped_column(Text(), nullable=True) + description: Mapped[str | None] = mapped_column(Text(), nullable=True) + image_url: Mapped[str | None] = mapped_column(Text(), nullable=True) + site_name: Mapped[str | None] = mapped_column(Text(), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) diff --git a/src/thoughtsync/notes.py b/src/thoughtsync/notes.py index bbe6acf..31f4a9a 100644 --- a/src/thoughtsync/notes.py +++ b/src/thoughtsync/notes.py @@ -18,11 +18,13 @@ from .auth import login_required from .config import Config from .db import session_scope from .settings import get_setting +from .unfurl import UnfurlError, unfurl from .models.label import Label, NoteLabel from .models.note import NOTE_COLORS, Note from .models.note_attachment import NoteAttachment from .models.note_item import NoteItem from .models.note_link import NoteLink +from .models.note_link_preview import NoteLinkPreview from .models.note_revision import NoteRevision ALLOWED_IMAGE_MIMES = {"image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif", "image/webp": ".webp"} @@ -173,6 +175,34 @@ async def _attachments_for_notes(db, note_ids: list) -> dict: return result +def _serialize_preview(p: NoteLinkPreview) -> dict: + return { + "id": str(p.id), + "url": p.url, + "title": p.title, + "description": p.description, + "image_url": p.image_url, + "site_name": p.site_name, + } + + +async def _previews_for_notes(db, note_ids: list) -> dict: + """Map note_id -> [link previews] in one query.""" + result: dict = {} + if not note_ids: + return result + rows = ( + await db.scalars( + select(NoteLinkPreview) + .where(NoteLinkPreview.note_id.in_(note_ids)) + .order_by(NoteLinkPreview.created_at) + ) + ).all() + for p in rows: + result.setdefault(p.note_id, []).append(_serialize_preview(p)) + return result + + async def _serialize_note(db, note: Note) -> dict: data = note.serialize() labels = await _labels_for_notes(db, [note.id]) @@ -181,6 +211,8 @@ async def _serialize_note(db, note: Note) -> dict: data["items"] = items.get(note.id, []) attachments = await _attachments_for_notes(db, [note.id]) data["attachments"] = attachments.get(note.id, []) + previews = await _previews_for_notes(db, [note.id]) + data["previews"] = previews.get(note.id, []) return data @@ -189,12 +221,14 @@ async def _serialize_notes(db, notes: list) -> list: labels_map = await _labels_for_notes(db, ids) items_map = await _items_for_notes(db, ids) attach_map = await _attachments_for_notes(db, ids) + preview_map = await _previews_for_notes(db, ids) out = [] for n in notes: data = n.serialize() data["labels"] = labels_map.get(n.id, []) data["items"] = items_map.get(n.id, []) data["attachments"] = attach_map.get(n.id, []) + data["previews"] = preview_map.get(n.id, []) out.append(data) return out @@ -1320,6 +1354,67 @@ async def delete_attachment(note_id: str, att_id: str): return jsonify(result) +@bp.post("/<note_id>/unfurl") +@login_required +async def unfurl_link(note_id: str): + """Fetch a link preview for a URL in this note and store it. Opt-in via the + enable_url_unfurl setting; SSRF-guarded server-side fetch (see unfurl.py).""" + data = await request.get_json(silent=True) or {} + url = (data.get("url") or "").strip() + if not url: + return jsonify({"error": "url is required"}), 400 + async with session_scope() as db: + note = await _get_owned(db, note_id) + if note is None: + return jsonify({"error": "not found"}), 404 + if not await get_setting(db, "enable_url_unfurl"): + return jsonify({"error": "link previews are disabled"}), 403 + # Fetch OUTSIDE the DB session — network IO shouldn't hold a connection. + try: + preview = await unfurl(url) + except UnfurlError as e: + return jsonify({"error": str(e)}), 502 + async with session_scope() as db: + note = await _get_owned(db, note_id) + if note is None: + return jsonify({"error": "not found"}), 404 + # Keyed by the ORIGINAL pasted url (what the note body contains, so the client + # matches it) — re-unfurling the same link updates the cached preview in place. + row = await db.scalar( + select(NoteLinkPreview).where(NoteLinkPreview.note_id == note.id, NoteLinkPreview.url == url) + ) + if row is None: + row = NoteLinkPreview(note_id=note.id, url=url) + db.add(row) + row.title = preview["title"] + row.description = preview["description"] + row.image_url = preview["image_url"] + row.site_name = preview["site_name"] + await db.commit() + return jsonify(await _serialize_note(db, note)), 201 + + +@bp.delete("/<note_id>/previews/<preview_id>") +@login_required +async def delete_preview(note_id: str, preview_id: str): + try: + pid = uuid.UUID(preview_id) + except (ValueError, TypeError): + return jsonify({"error": "not found"}), 404 + async with session_scope() as db: + note = await _get_owned(db, note_id) + if note is None: + return jsonify({"error": "not found"}), 404 + row = await db.scalar( + select(NoteLinkPreview).where(NoteLinkPreview.id == pid, NoteLinkPreview.note_id == note.id) + ) + if row is None: + return jsonify({"error": "not found"}), 404 + await db.delete(row) + await db.commit() + return jsonify(await _serialize_note(db, note)) + + @bp.post("/<note_id>/trash") @login_required async def trash_note(note_id: str): diff --git a/src/thoughtsync/settings.py b/src/thoughtsync/settings.py index 8f4d4d7..fec3a8e 100644 --- a/src/thoughtsync/settings.py +++ b/src/thoughtsync/settings.py @@ -51,6 +51,15 @@ REGISTRY: list[SettingDef] = [ "Largest single file that can be attached to a note. Capped by the server body limit.", "Attachments", ), + SettingDef( + "enable_url_unfurl", + "bool", + True, + "Link previews", + "Let the server fetch a page's title/description/image to preview pasted links. " + "The server contacts the linked site; private/internal addresses are always blocked.", + "Links", + ), ] _BY_KEY: dict[str, SettingDef] = {d.key: d for d in REGISTRY} @@ -112,6 +121,7 @@ async def get_public_config(db) -> dict: return { "site_name": await get_setting(db, "site_name"), "allow_registration": await get_setting(db, "allow_registration"), + "enable_url_unfurl": await get_setting(db, "enable_url_unfurl"), } diff --git a/src/thoughtsync/unfurl.py b/src/thoughtsync/unfurl.py new file mode 100644 index 0000000..eaa7394 --- /dev/null +++ b/src/thoughtsync/unfurl.py @@ -0,0 +1,194 @@ +"""SSRF-hardened link unfurling — fetch a URL server-side and extract an OG/meta +preview. Dependency-free (stdlib only), matching the project's hand-rolled ethos. + +The server fetching arbitrary user-supplied URLs is a classic SSRF surface, so the +defenses are deliberate and layered: + - http/https only (no file://, gopher://, …). + - Resolve the host and require EVERY resolved address to be public — reject + private / loopback / link-local / reserved / multicast / unspecified ranges. + - Connect to the exact vetted IP (with SNI = the hostname), so a name that + re-resolves to an internal address between check and connect (DNS rebinding) + can't slip through. + - At most 3 redirects, each hop re-validated the same way. + - 5s timeout, 512 KB body cap, text/html only. +No AI — this just parses OpenGraph/Twitter/`<title>` meta tags. +""" +from __future__ import annotations + +import asyncio +import http.client +import ipaddress +import re +import socket +import ssl +from html import unescape +from urllib.parse import urljoin, urlparse + +MAX_REDIRECTS = 3 +TIMEOUT_S = 5.0 +MAX_BYTES = 512 * 1024 +USER_AGENT = "ThoughtSync-LinkPreview/1.0" +_REDIRECT_CODES = {301, 302, 303, 307, 308} + + +class UnfurlError(Exception): + """A URL could not be safely unfurled (bad scheme, blocked address, fetch error).""" + + +def is_public_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """True only for globally-routable addresses — everything internal is rejected.""" + return not ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ) + + +def validate_url(raw: str) -> tuple[str, str, int, str]: + """Parse a safe http(s) URL → (scheme, host, port, path+query). Raise otherwise.""" + parsed = urlparse((raw or "").strip()) + if parsed.scheme not in ("http", "https"): + raise UnfurlError("only http and https links can be previewed") + host = parsed.hostname + if not host: + raise UnfurlError("that link has no host") + port = parsed.port or (443 if parsed.scheme == "https" else 80) + path = parsed.path or "/" + if parsed.query: + path = f"{path}?{parsed.query}" + return parsed.scheme, host, port, path + + +def _resolve_public(host: str, port: int) -> str: + """Resolve host; require ALL resolved addresses to be public. Return one vetted IP.""" + try: + infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) + except socket.gaierror as e: + raise UnfurlError("could not resolve that host") from e + chosen: str | None = None + for info in infos: + addr = info[4][0] + try: + ip = ipaddress.ip_address(addr.split("%")[0]) # strip any IPv6 zone id + except ValueError as e: + raise UnfurlError("could not resolve that host") from e + if not is_public_ip(ip): + raise UnfurlError("that address isn't allowed") + if chosen is None: + chosen = addr + if chosen is None: + raise UnfurlError("could not resolve that host") + return chosen + + +def _fetch_once(scheme: str, host: str, port: int, path: str) -> tuple[int, str | None, bytes]: + """One blocking GET to the VETTED public IP for `host`. Returns (status, location, + body). Reads at most MAX_BYTES of a text/html body.""" + ip = _resolve_public(host, port) + sock: socket.socket = socket.create_connection((ip, port), timeout=TIMEOUT_S) + try: + if scheme == "https": + ctx = ssl.create_default_context() + sock = ctx.wrap_socket(sock, server_hostname=host) # SNI + cert check vs host + sock.settimeout(TIMEOUT_S) # ensure reads can't hang after the TLS wrap + conn = http.client.HTTPConnection(host, port, timeout=TIMEOUT_S) + conn.sock = sock # use our pre-vetted (and, for https, wrapped) socket + conn.request( + "GET", + path, + headers={ + "User-Agent": USER_AGENT, + "Accept": "text/html,application/xhtml+xml", + "Accept-Encoding": "identity", + "Connection": "close", + }, + ) + resp = conn.getresponse() + headers = {k.lower(): v for k, v in resp.getheaders()} + if resp.status in _REDIRECT_CODES: + return resp.status, headers.get("location"), b"" + ctype = headers.get("content-type", "").split(";")[0].strip().lower() + if ctype and ctype not in ("text/html", "application/xhtml+xml"): + raise UnfurlError("that link isn't a web page") + return resp.status, None, resp.read(MAX_BYTES) + finally: + try: + sock.close() + except OSError: + pass + + +def _fetch(url: str) -> tuple[str, bytes]: + """Follow up to MAX_REDIRECTS, re-validating each hop. Returns (final_url, html).""" + current = url + for _ in range(MAX_REDIRECTS + 1): + scheme, host, port, path = validate_url(current) + try: + status, location, body = _fetch_once(scheme, host, port, path) + except UnfurlError: + raise + except (OSError, ssl.SSLError, http.client.HTTPException) as e: + raise UnfurlError("could not fetch that link") from e + if status in _REDIRECT_CODES and location: + current = urljoin(current, location) + continue + if status >= 400: + raise UnfurlError(f"the site returned an error ({status})") + return current, body + raise UnfurlError("too many redirects") + + +def _meta_content(html: str, key: str) -> str | None: + """The content of a <meta property|name="key" content="…"> tag (either attr order).""" + k = re.escape(key) + for pat in ( + rf'<meta[^>]+(?:property|name)=["\']{k}["\'][^>]*content=["\']([^"\']*)["\']', + rf'<meta[^>]+content=["\']([^"\']*)["\'][^>]*(?:property|name)=["\']{k}["\']', + ): + m = re.search(pat, html, re.IGNORECASE | re.DOTALL) + if m: + val = unescape(m.group(1)).strip() + if val: + return val + return None + + +def extract_preview(final_url: str, body: bytes) -> dict: + """Pull a link preview (title/description/image/site) from HTML. OpenGraph first, + then Twitter cards, then <title> / bare <meta name=description>.""" + html = body.decode("utf-8", errors="replace") + title = _meta_content(html, "og:title") or _meta_content(html, "twitter:title") + if not title: + tm = re.search(r"<title[^>]*>(.*?)", html, re.IGNORECASE | re.DOTALL) + if tm: + title = unescape(re.sub(r"\s+", " ", tm.group(1)).strip()) or None + description = ( + _meta_content(html, "og:description") + or _meta_content(html, "twitter:description") + or _meta_content(html, "description") + ) + image = _meta_content(html, "og:image") or _meta_content(html, "twitter:image") + if image: + image = urljoin(final_url, image) # resolve a relative image path + if urlparse(image).scheme not in ("http", "https"): + image = None + site_name = _meta_content(html, "og:site_name") + host = urlparse(final_url).hostname or "" + return { + "url": final_url, + "title": (title or host)[:300], + "description": description[:500] if description else None, + "image_url": image[:1000] if image else None, + "site_name": (site_name or host)[:100] or None, + } + + +async def unfurl(url: str) -> dict: + """Fetch `url` server-side (SSRF-guarded) and return a link preview. The blocking + socket IO runs in a worker thread so it never stalls the event loop. Raises + UnfurlError on any failure.""" + final_url, body = await asyncio.to_thread(_fetch, url) + return extract_preview(final_url, body) diff --git a/tests/test_unfurl.py b/tests/test_unfurl.py new file mode 100644 index 0000000..a5d844d --- /dev/null +++ b/tests/test_unfurl.py @@ -0,0 +1,74 @@ +import ipaddress + +import pytest + +from thoughtsync.app import create_app +from thoughtsync.unfurl import UnfurlError, extract_preview, is_public_ip, validate_url + + +@pytest.fixture +def app(): + return create_app() + + +def test_is_public_ip_blocks_internal_ranges(): + assert is_public_ip(ipaddress.ip_address("8.8.8.8")) + assert is_public_ip(ipaddress.ip_address("2606:4700:4700::1111")) + # everything internal / special is rejected (the SSRF core) + assert not is_public_ip(ipaddress.ip_address("10.0.0.1")) # private + assert not is_public_ip(ipaddress.ip_address("192.168.1.1")) # private + assert not is_public_ip(ipaddress.ip_address("127.0.0.1")) # loopback + assert not is_public_ip(ipaddress.ip_address("169.254.169.254")) # link-local (cloud metadata) + assert not is_public_ip(ipaddress.ip_address("0.0.0.0")) # unspecified + assert not is_public_ip(ipaddress.ip_address("::1")) # loopback v6 + assert not is_public_ip(ipaddress.ip_address("fc00::1")) # unique-local v6 + + +def test_validate_url_scheme_and_parts(): + assert validate_url("https://example.com/a?b=c") == ("https", "example.com", 443, "/a?b=c") + assert validate_url("http://x.io")[3] == "/" # default path + assert validate_url("http://x.io:8080/p")[2] == 8080 # explicit port + for bad in ("file:///etc/passwd", "ftp://x", "gopher://x", "not a url", ""): + with pytest.raises(UnfurlError): + validate_url(bad) + + +def test_extract_preview_opengraph(): + html = ( + b"" + b'' + b'' + b'' + b'' + b"" + ) + p = extract_preview("https://example.com/page", html) + assert p["title"] == "Hello & World" # entities decoded + assert p["description"] == "A page" + assert p["image_url"] == "https://example.com/img.png" # relative resolved to absolute + assert p["site_name"] == "Example" + + +def test_extract_preview_title_fallback_and_host_defaults(): + html = b" Just a Title " + p = extract_preview("https://example.com", html) + assert p["title"] == "Just a Title" # whitespace collapsed + assert p["description"] is None + assert p["image_url"] is None + assert p["site_name"] == "example.com" # falls back to host + + +async def test_unfurl_requires_auth(app): + client = app.test_client() + resp = await client.post( + "/api/notes/00000000-0000-0000-0000-000000000000/unfurl", json={"url": "https://x.com"} + ) + assert resp.status_code == 401 + + +async def test_delete_preview_requires_auth(app): + client = app.test_client() + resp = await client.delete( + "/api/notes/00000000-0000-0000-0000-000000000000/previews/00000000-0000-0000-0000-000000000001" + ) + assert resp.status_code == 401