bvandeusen
d96918d777
Capture off-platform links (mega/gdrive/mediafire/dropbox/pixeldrain) embedded in post bodies so they're never silently dropped, and surface them in the post view. The download worker (Phase 4) walks these rows. - link_extract.py: pure extractor — <a href> + bare URLs, unwraps Patreon redirect shims, PRESERVES the full url incl. #fragment (mega's key), dedups. Reusable by every platform (runs off Post.description). - external_link model + migration 0049: post_id/artist_id/host/url/label/status /attempts/last_error/attachment_id/timing; CHECK whitelists (full enum incl. worker statuses up front) + (post_id,url) unique. - importer._sync_external_links: insert-missing on both import paths (_apply_sidecar + upsert_post_record) so a re-import never resets a link's status; runs for all platforms. - post_feed_service.get_post: returns external_links (detail-only). - PostCard: renders the links (host chip + label + status) once expanded. - tests: extractor (5 hosts, fragment, shim unwrap, dedup), importer (record + no-dup on reimport), serializer. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
77 lines
3.0 KiB
Python
77 lines
3.0 KiB
Python
"""Unit tests for link_extract — pure, no DB/network."""
|
|
|
|
from backend.app.services.link_extract import extract_external_links, host_for
|
|
|
|
|
|
def test_host_classification_covers_all_and_rejects_others():
|
|
assert host_for("https://mega.nz/file/x#k") == "mega"
|
|
assert host_for("https://www.dropbox.com/s/x?dl=0") == "dropbox"
|
|
assert host_for("https://drive.google.com/file/d/ID/view") == "gdrive"
|
|
assert host_for("https://pixeldrain.com/u/abc") == "pixeldrain"
|
|
assert host_for("https://www.mediafire.com/file/x") == "mediafire"
|
|
assert host_for("https://example.com/x") is None
|
|
|
|
|
|
def test_anchor_preserves_fragment_and_label():
|
|
html = ('<p>Watch: '
|
|
'<a href="https://mega.nz/file/Bux0FRrZ#b2Pn9841">Mega - Streamable</a></p>')
|
|
links = extract_external_links(html)
|
|
assert len(links) == 1
|
|
assert links[0].host == "mega"
|
|
# The #fragment (mega's decryption key) MUST survive.
|
|
assert links[0].url == "https://mega.nz/file/Bux0FRrZ#b2Pn9841"
|
|
assert links[0].label == "Mega - Streamable"
|
|
|
|
|
|
def test_extracts_all_five_hosts():
|
|
html = (
|
|
'<a href="https://mega.nz/file/a#k">m</a>'
|
|
'<a href="https://drive.google.com/file/d/ID/view?usp=sharing">g</a>'
|
|
'<a href="https://www.mediafire.com/file/x/y">mf</a>'
|
|
'<a href="https://www.dropbox.com/s/x/y?dl=0">db</a>'
|
|
'<a href="https://pixeldrain.com/u/ems4UomN">px</a>'
|
|
)
|
|
hosts = {lnk.host for lnk in extract_external_links(html)}
|
|
assert hosts == {"mega", "gdrive", "mediafire", "dropbox", "pixeldrain"}
|
|
|
|
|
|
def test_bare_url_in_text():
|
|
html = "Full film: https://pixeldrain.com/u/ems4UomN and more prose."
|
|
links = extract_external_links(html)
|
|
assert len(links) == 1
|
|
assert links[0].host == "pixeldrain"
|
|
assert links[0].url == "https://pixeldrain.com/u/ems4UomN" # trailing '.' trimmed
|
|
assert links[0].label is None
|
|
|
|
|
|
def test_unwraps_patreon_redirect():
|
|
html = ('<a href="https://www.patreon.com/redirect?'
|
|
'url=https%3A%2F%2Fmega.nz%2Ffile%2Fz%23key">x</a>')
|
|
links = extract_external_links(html)
|
|
assert len(links) == 1
|
|
assert links[0].host == "mega"
|
|
assert links[0].url == "https://mega.nz/file/z#key"
|
|
|
|
|
|
def test_ignores_unsupported_and_dedups_anchor_label_wins():
|
|
html = (
|
|
'<a href="https://example.com/x">no</a>'
|
|
'<a href="https://mega.nz/file/dup#k">a</a>'
|
|
' bare https://mega.nz/file/dup#k'
|
|
)
|
|
links = extract_external_links(html)
|
|
assert len(links) == 1 # example.com dropped; mega dup collapsed
|
|
assert links[0].label == "a" # anchor (with label) wins over bare
|
|
|
|
|
|
def test_amp_escaped_query_is_unescaped():
|
|
html = '<a href="https://drive.google.com/uc?id=ID&export=download">g</a>'
|
|
links = extract_external_links(html)
|
|
assert links[0].url == "https://drive.google.com/uc?id=ID&export=download"
|
|
|
|
|
|
def test_empty_inputs():
|
|
assert extract_external_links(None) == []
|
|
assert extract_external_links("") == []
|
|
assert extract_external_links("<p>no links here</p>") == []
|