"""Extract off-platform file-host links from a post body.
Pure (no I/O) so it's unit-testable and reusable by EVERY in-house downloader's
import path — every platform stores its body in `Post.description`, so running
this there covers them all. Finds links to the supported external file hosts in
a post's HTML body (both `` anchors and bare URLs in text),
unwraps a Patreon outbound-redirect wrapper, and preserves the FULL url
including the `#fragment` (mega.nz puts the decryption key there) and the query
string — without those a mega download is impossible.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from html import unescape
from urllib.parse import parse_qs, urlsplit
# Supported file-host enum values (kept in sync with the external_link CHECK).
SUPPORTED_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
# Bare-domain suffix → canonical host label. Matched against the URL netloc
# (exact or as a dotted suffix, so www./dl. subdomains resolve too).
_HOST_MAP = {
"mega.nz": "mega",
"mega.co.nz": "mega",
"drive.google.com": "gdrive",
"mediafire.com": "mediafire",
"dropbox.com": "dropbox",
"pixeldrain.com": "pixeldrain",
}
# `label` — DOTALL so a label spanning tags/newlines is caught.
_HREF_RE = re.compile(
r"""]*?\bhref=["']([^"']+)["'][^>]*>(.*?)""",
re.IGNORECASE | re.DOTALL,
)
_TAG_RE = re.compile(r"<[^>]+>")
# Bare http(s) URL in text. Stops at whitespace, quotes, angle brackets, and
# closing brackets — but KEEPS `#`, `&`, `?`, `=` so fragments/queries survive.
_URL_RE = re.compile(r"""https?://[^\s"'<>)\]}]+""", re.IGNORECASE)
@dataclass(frozen=True)
class ExtractedLink:
host: str # one of SUPPORTED_HOSTS
url: str # full url incl. query + #fragment
label: str | None # visible anchor text, when present
def _netloc(url: str) -> str:
# Lowercase host without credentials or port.
return urlsplit(url).netloc.lower().split("@")[-1].split(":")[0]
def host_for(url: str) -> str | None:
"""Canonical host label for a url, or None if it's not a supported host."""
netloc = _netloc(url)
for suffix, host in _HOST_MAP.items():
if netloc == suffix or netloc.endswith("." + suffix):
return host
return None
def _unwrap(url: str) -> str:
"""Unwrap a Patreon outbound-redirect wrapper to its inner target, so a
wrapped mega/gdrive link resolves to the real host. Patreon has used both
`www.patreon.com/...?url=` and the `l.patreon.com` shim; check the
common target-param names. Non-Patreon urls pass through untouched."""
netloc = _netloc(url)
if not (netloc == "patreon.com" or netloc.endswith(".patreon.com")):
return url
qs = parse_qs(urlsplit(url).query)
for key in ("url", "u", "ext_url", "redirect", "target"):
vals = qs.get(key)
if vals and vals[0]:
return unescape(vals[0]).strip()
return url
def extract_external_links(html: str | None) -> list[ExtractedLink]:
"""All supported-host links in `html`, de-duplicated by url (first wins, so
an anchor's label is kept over a later bare sighting of the same url)."""
if not html:
return []
found: dict[str, ExtractedLink] = {}
# 1) Anchors first — they carry a human label ("Mega - Streamable").
for raw_href, inner in _HREF_RE.findall(html):
url = _unwrap(unescape(raw_href).strip())
host = host_for(url)
if host is None:
continue
label = unescape(_TAG_RE.sub("", inner)).strip() or None
found.setdefault(url, ExtractedLink(host=host, url=url, label=label))
# 2) Bare URLs pasted in text (no label). Trailing prose punctuation is
# trimmed; the href values already captured above de-dup away here.
for raw in _URL_RE.findall(html):
url = _unwrap(unescape(raw).strip().rstrip(".,;"))
host = host_for(url)
if host is None:
continue
found.setdefault(url, ExtractedLink(host=host, url=url, label=None))
return list(found.values())