diff --git a/alembic/versions/0049_external_link_table.py b/alembic/versions/0049_external_link_table.py new file mode 100644 index 0000000..373c807 --- /dev/null +++ b/alembic/versions/0049_external_link_table.py @@ -0,0 +1,90 @@ +"""external_link table — off-platform file-host links found in post bodies + +Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox / +Pixeldrain and link them in the post text. This table records each such link +(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger +the download worker (a later slice) walks. `url` keeps the FULL link including +the `#fragment` — mega.nz's decryption key lives there; truncating it makes the +file undownloadable. + +CHECK whitelists for host + status include the full enum up front (incl. the +download-worker statuses) so the worker slice needs no constraint migration. + +Revision ID: 0049 +Revises: 0048 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0049" +down_revision: Union[str, None] = "0048" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "external_link", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "post_id", sa.Integer(), + sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False, + ), + sa.Column( + "artist_id", sa.Integer(), + sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column("host", sa.String(length=16), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("label", sa.Text(), nullable=True), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="pending", + ), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "attachment_id", sa.Integer(), + sa.ForeignKey("post_attachment.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("duration_seconds", sa.Float(), nullable=True), + sa.CheckConstraint( + "host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')", + name="ck_external_link_host", + ), + sa.CheckConstraint( + "status IN ('pending','downloading','downloaded','failed'," + "'skipped','dead')", + name="ck_external_link_status", + ), + ) + op.create_index( + "ix_external_link_post_id", "external_link", ["post_id"], + ) + op.create_index( + "ix_external_link_artist_id", "external_link", ["artist_id"], + ) + op.create_index( + "ix_external_link_status", "external_link", ["status"], + ) + op.create_index( + "uq_external_link_post_url", "external_link", ["post_id", "url"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("uq_external_link_post_url", table_name="external_link") + op.drop_index("ix_external_link_status", table_name="external_link") + op.drop_index("ix_external_link_artist_id", table_name="external_link") + op.drop_index("ix_external_link_post_id", table_name="external_link") + op.drop_table("external_link") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 19d444b..4433dc2 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -7,6 +7,7 @@ from .backup_run import BackupRun from .base import Base from .credential import Credential from .download_event import DownloadEvent +from .external_link import ExternalLink from .image_prediction import ImagePrediction from .image_provenance import ImageProvenance from .image_record import ImageRecord @@ -52,6 +53,7 @@ __all__ = [ "TagKind", "image_tag", "DownloadEvent", + "ExternalLink", "ImportBatch", "ImportTask", "ImportSettings", diff --git a/backend/app/models/external_link.py b/backend/app/models/external_link.py new file mode 100644 index 0000000..0902e28 --- /dev/null +++ b/backend/app/models/external_link.py @@ -0,0 +1,73 @@ +"""ExternalLink — an off-platform file-host link found in a post body. + +Creators host the actual files (films, packs) on mega.nz / Google Drive / +MediaFire / Dropbox / Pixeldrain and drop the link in the post text. This row +is the record that the link existed (so nothing is silently dropped), the +dedup + dead-letter ledger for fetching it, and the driver the download worker +walks. `url` keeps the FULL link including the `#fragment` (mega's decryption +key) — truncating it makes the file undownloadable. + +status lifecycle: pending → downloading → downloaded | failed | dead +(too many attempts) | skipped (host disabled). `attachment_id` links the +captured file once a download lands (SET NULL so deleting the attachment +doesn't delete the link record). +""" + +from datetime import datetime + +from sqlalchemy import ( + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + func, + text, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + +# Kept in sync with link_extract.SUPPORTED_HOSTS and the CHECK in migration 0049. +HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain") +STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead") + + +class ExternalLink(Base): + __tablename__ = "external_link" + __table_args__ = ( + # One row per (post, url). The full url (incl. #fragment) is the identity + # — the same file linked twice in a post collapses to one row. + Index("uq_external_link_post_url", "post_id", "url", unique=True), + Index("ix_external_link_status", "status"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + artist_id: Mapped[int | None] = mapped_column( + ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True + ) + host: Mapped[str] = mapped_column(String(16), nullable=False) + url: Mapped[str] = mapped_column(Text, nullable=False) + label: Mapped[str | None] = mapped_column(Text, nullable=True) + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="pending" + ) + attempts: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + attachment_id: Mapped[int | None] = mapped_column( + ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + completed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True) diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 280b3be..c412ee3 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -23,6 +23,7 @@ from sqlalchemy.orm import Session from ..models import ( Artist, + ExternalLink, ImageProvenance, ImageRecord, ImportSettings, @@ -39,6 +40,7 @@ from ..utils.paths import ( ) from ..utils.phash import compute_phash, find_similar from ..utils.sidecar import find_sidecar, parse_sidecar +from .link_extract import extract_external_links from ..utils.slug import slugify from .archive_extractor import extract_archive, is_archive from .attachment_store import AttachmentStore @@ -305,6 +307,32 @@ class Importer: ), ) + def _sync_external_links(self, post: Post) -> None: + """Record off-platform file-host links (mega/gdrive/mediafire/dropbox/ + pixeldrain) found in the post body, so they're never silently dropped + and the download worker can fetch them later. Shared by every platform's + import (runs off Post.description). INSERT-MISSING only — an existing row + keeps its status/attempts (a re-import must not reset a link already + downloaded or dead-lettered). Identity is (post_id, url); the full url + incl. #fragment is preserved by the extractor.""" + links = extract_external_links(post.description) + if not links: + return + self.session.flush() # ensure post.id is assigned before we reference it + existing = set(self.session.execute( + select(ExternalLink.url).where(ExternalLink.post_id == post.id) + ).scalars().all()) + for link in links: + if link.url in existing: + continue + self.session.add(ExternalLink( + post_id=post.id, + artist_id=post.artist_id, + host=link.host, + url=link.url, + label=link.label, + )) + def import_one(self, source: Path) -> ImportResult: """Dispatch by kind. Media → normal pipeline. Archive → extract media members (one Post via the archive-adjacent sidecar) and @@ -767,6 +795,7 @@ class Importer: if sd.attachment_count is not None: post.attachment_count = sd.attachment_count post.raw_metadata = sd.raw + self._sync_external_links(post) self.session.commit() return True @@ -1042,6 +1071,7 @@ class Importer: if sd.attachment_count is not None: post.attachment_count = sd.attachment_count post.raw_metadata = sd.raw + self._sync_external_links(post) # Race-safe (image_record_id, post_id) upsert — mirrors the # _find_or_create_source/post savepoint pattern. The plain diff --git a/backend/app/services/link_extract.py b/backend/app/services/link_extract.py new file mode 100644 index 0000000..e6ac046 --- /dev/null +++ b/backend/app/services/link_extract.py @@ -0,0 +1,105 @@ +"""Extract off-platform file-host links from a post body. + +Pure (no I/O) so it's unit-testable and reusable by EVERY in-house downloader's +import path — every platform stores its body in `Post.description`, so running +this there covers them all. Finds links to the supported external file hosts in +a post's HTML body (both `` anchors and bare URLs in text), +unwraps a Patreon outbound-redirect wrapper, and preserves the FULL url +including the `#fragment` (mega.nz puts the decryption key there) and the query +string — without those a mega download is impossible. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from html import unescape +from urllib.parse import parse_qs, urlsplit + +# Supported file-host enum values (kept in sync with the external_link CHECK). +SUPPORTED_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain") + +# Bare-domain suffix → canonical host label. Matched against the URL netloc +# (exact or as a dotted suffix, so www./dl. subdomains resolve too). +_HOST_MAP = { + "mega.nz": "mega", + "mega.co.nz": "mega", + "drive.google.com": "gdrive", + "mediafire.com": "mediafire", + "dropbox.com": "dropbox", + "pixeldrain.com": "pixeldrain", +} + +# `label` — DOTALL so a label spanning tags/newlines is caught. +_HREF_RE = re.compile( + r"""]*?\bhref=["']([^"']+)["'][^>]*>(.*?)""", + re.IGNORECASE | re.DOTALL, +) +_TAG_RE = re.compile(r"<[^>]+>") +# Bare http(s) URL in text. Stops at whitespace, quotes, angle brackets, and +# closing brackets — but KEEPS `#`, `&`, `?`, `=` so fragments/queries survive. +_URL_RE = re.compile(r"""https?://[^\s"'<>)\]}]+""", re.IGNORECASE) + + +@dataclass(frozen=True) +class ExtractedLink: + host: str # one of SUPPORTED_HOSTS + url: str # full url incl. query + #fragment + label: str | None # visible anchor text, when present + + +def _netloc(url: str) -> str: + # Lowercase host without credentials or port. + return urlsplit(url).netloc.lower().split("@")[-1].split(":")[0] + + +def host_for(url: str) -> str | None: + """Canonical host label for a url, or None if it's not a supported host.""" + netloc = _netloc(url) + for suffix, host in _HOST_MAP.items(): + if netloc == suffix or netloc.endswith("." + suffix): + return host + return None + + +def _unwrap(url: str) -> str: + """Unwrap a Patreon outbound-redirect wrapper to its inner target, so a + wrapped mega/gdrive link resolves to the real host. Patreon has used both + `www.patreon.com/...?url=` and the `l.patreon.com` shim; check the + common target-param names. Non-Patreon urls pass through untouched.""" + netloc = _netloc(url) + if not (netloc == "patreon.com" or netloc.endswith(".patreon.com")): + return url + qs = parse_qs(urlsplit(url).query) + for key in ("url", "u", "ext_url", "redirect", "target"): + vals = qs.get(key) + if vals and vals[0]: + return unescape(vals[0]).strip() + return url + + +def extract_external_links(html: str | None) -> list[ExtractedLink]: + """All supported-host links in `html`, de-duplicated by url (first wins, so + an anchor's label is kept over a later bare sighting of the same url).""" + if not html: + return [] + found: dict[str, ExtractedLink] = {} + + # 1) Anchors first — they carry a human label ("Mega - Streamable"). + for raw_href, inner in _HREF_RE.findall(html): + url = _unwrap(unescape(raw_href).strip()) + host = host_for(url) + if host is None: + continue + label = unescape(_TAG_RE.sub("", inner)).strip() or None + found.setdefault(url, ExtractedLink(host=host, url=url, label=label)) + + # 2) Bare URLs pasted in text (no label). Trailing prose punctuation is + # trimmed; the href values already captured above de-dup away here. + for raw in _URL_RE.findall(html): + url = _unwrap(unescape(raw).strip().rstrip(".,;")) + host = host_for(url) + if host is None: + continue + found.setdefault(url, ExtractedLink(host=host, url=url, label=None)) + + return list(found.values()) diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index 07d5b68..ad053d4 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from ..models import ( Artist, + ExternalLink, ImageProvenance, ImageRecord, Post, @@ -196,8 +197,26 @@ class PostFeedService: # detail-only (the feed list stays lightweight plain text). None when the # post has no body. item["description_html"] = sanitize_post_html(post.description) + item["external_links"] = await self._external_links_for(post.id) return item + async def _external_links_for(self, post_id: int) -> list[dict]: + """Off-platform file-host links recorded for a post (detail-only). Each + carries its host, full url, label, and download status so the post view + can surface them (and, later, a retry/download affordance).""" + rows = (await self.session.execute( + select(ExternalLink) + .where(ExternalLink.post_id == post_id) + .order_by(ExternalLink.id.asc()) + )).scalars().all() + return [ + { + "id": e.id, "host": e.host, "url": e.url, + "label": e.label, "status": e.status, + } + for e in rows + ] + # --- composition helpers --------------------------------------------- async def _thumbnails_for( diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index 0a3728f..29f15f9 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -87,6 +87,25 @@ @click="toggleDesc" >{{ descExpanded ? 'Show less' : 'Show more' }} + +
+
External files
+ + {{ lnk.host }} + {{ lnk.label || lnk.url }} + {{ lnk.status }} + +
+
detail.value?.description_full || null) // Backend-sanitized HTML body (detail-only). Drives the faithful render once // expanded; falls back to plain text when detail isn't loaded. const descHtml = computed(() => detail.value?.description_html || null) +// Off-platform file-host links (detail-only), surfaced once expanded. +const externalLinks = computed(() => detail.value?.external_links || []) const descText = computed(() => descExpanded.value ? (fullDescription.value || props.post.description_plain) @@ -469,4 +490,46 @@ function formatBytes (n) { } .fc-post-card__att-icon { color: rgb(var(--v-theme-on-surface-variant)); } .fc-post-card__att-size { color: rgb(var(--v-theme-on-surface-variant)); } + +.fc-post-card__links { + display: flex; flex-direction: column; gap: 6px; + margin-top: 12px; +} +.fc-post-card__links-label { + font-size: 0.75rem; font-weight: 600; text-transform: uppercase; + letter-spacing: 0.04em; + color: rgb(var(--v-theme-on-surface-variant)); +} +.fc-post-card__link { + display: inline-flex; align-items: center; gap: 8px; + padding: 6px 12px; + border: 1px solid rgb(var(--v-theme-on-surface-variant)); + border-radius: 8px; + color: rgb(var(--v-theme-on-surface)); + text-decoration: none; + font-size: 0.85rem; +} +.fc-post-card__link:hover { + color: rgb(var(--v-theme-accent)); + border-color: rgb(var(--v-theme-accent)); +} +.fc-post-card__link-host { + flex: 0 0 auto; + font-weight: 700; text-transform: uppercase; font-size: 0.7rem; + letter-spacing: 0.04em; + padding: 2px 6px; border-radius: 4px; + background: rgba(var(--v-theme-accent), 0.16); + color: rgb(var(--v-theme-accent)); +} +.fc-post-card__link-text { + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; +} +.fc-post-card__link-status { + flex: 0 0 auto; margin-left: auto; + font-size: 0.7rem; text-transform: uppercase; + color: rgb(var(--v-theme-on-surface-variant)); +} +.fc-post-card__link-status[data-status="downloaded"] { + color: rgb(var(--v-theme-accent)); +} diff --git a/tests/test_link_extract.py b/tests/test_link_extract.py new file mode 100644 index 0000000..9f13c7d --- /dev/null +++ b/tests/test_link_extract.py @@ -0,0 +1,76 @@ +"""Unit tests for link_extract — pure, no DB/network.""" + +from backend.app.services.link_extract import extract_external_links, host_for + + +def test_host_classification_covers_all_and_rejects_others(): + assert host_for("https://mega.nz/file/x#k") == "mega" + assert host_for("https://www.dropbox.com/s/x?dl=0") == "dropbox" + assert host_for("https://drive.google.com/file/d/ID/view") == "gdrive" + assert host_for("https://pixeldrain.com/u/abc") == "pixeldrain" + assert host_for("https://www.mediafire.com/file/x") == "mediafire" + assert host_for("https://example.com/x") is None + + +def test_anchor_preserves_fragment_and_label(): + html = ('

Watch: ' + 'Mega - Streamable

') + links = extract_external_links(html) + assert len(links) == 1 + assert links[0].host == "mega" + # The #fragment (mega's decryption key) MUST survive. + assert links[0].url == "https://mega.nz/file/Bux0FRrZ#b2Pn9841" + assert links[0].label == "Mega - Streamable" + + +def test_extracts_all_five_hosts(): + html = ( + 'm' + 'g' + 'mf' + 'db' + 'px' + ) + hosts = {lnk.host for lnk in extract_external_links(html)} + assert hosts == {"mega", "gdrive", "mediafire", "dropbox", "pixeldrain"} + + +def test_bare_url_in_text(): + html = "Full film: https://pixeldrain.com/u/ems4UomN and more prose." + links = extract_external_links(html) + assert len(links) == 1 + assert links[0].host == "pixeldrain" + assert links[0].url == "https://pixeldrain.com/u/ems4UomN" # trailing '.' trimmed + assert links[0].label is None + + +def test_unwraps_patreon_redirect(): + html = ('x') + links = extract_external_links(html) + assert len(links) == 1 + assert links[0].host == "mega" + assert links[0].url == "https://mega.nz/file/z#key" + + +def test_ignores_unsupported_and_dedups_anchor_label_wins(): + html = ( + 'no' + 'a' + ' bare https://mega.nz/file/dup#k' + ) + links = extract_external_links(html) + assert len(links) == 1 # example.com dropped; mega dup collapsed + assert links[0].label == "a" # anchor (with label) wins over bare + + +def test_amp_escaped_query_is_unescaped(): + html = 'g' + links = extract_external_links(html) + assert links[0].url == "https://drive.google.com/uc?id=ID&export=download" + + +def test_empty_inputs(): + assert extract_external_links(None) == [] + assert extract_external_links("") == [] + assert extract_external_links("

no links here

") == [] diff --git a/tests/test_post_feed_service.py b/tests/test_post_feed_service.py index 064b000..7374743 100644 --- a/tests/test_post_feed_service.py +++ b/tests/test_post_feed_service.py @@ -11,6 +11,7 @@ from sqlalchemy import select from backend.app.models import ( Artist, + ExternalLink, ImageProvenance, ImageRecord, Post, @@ -437,6 +438,27 @@ async def test_get_post_returns_sanitized_html_body(db): assert "