feat(posts): extract + record external file-host links (Phase 3)
Capture off-platform links (mega/gdrive/mediafire/dropbox/pixeldrain) embedded in post bodies so they're never silently dropped, and surface them in the post view. The download worker (Phase 4) walks these rows. - link_extract.py: pure extractor — <a href> + bare URLs, unwraps Patreon redirect shims, PRESERVES the full url incl. #fragment (mega's key), dedups. Reusable by every platform (runs off Post.description). - external_link model + migration 0049: post_id/artist_id/host/url/label/status /attempts/last_error/attachment_id/timing; CHECK whitelists (full enum incl. worker statuses up front) + (post_id,url) unique. - importer._sync_external_links: insert-missing on both import paths (_apply_sidecar + upsert_post_record) so a re-import never resets a link's status; runs for all platforms. - post_feed_service.get_post: returns external_links (detail-only). - PostCard: renders the links (host chip + label + status) once expanded. - tests: extractor (5 hosts, fragment, shim unwrap, dedup), importer (record + no-dup on reimport), serializer. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ExternalLink,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
@@ -226,3 +227,47 @@ def test_upsert_post_record_idempotent_no_double(importer, import_layout):
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(Post)
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
def test_external_links_recorded_from_body(importer, import_layout):
|
||||
"""A mega/gdrive/etc. link in the body is recorded as an external_link row
|
||||
(status pending) so it's never silently dropped."""
|
||||
import_root, _ = import_layout
|
||||
artist = Artist(name="Linker", slug="linker")
|
||||
importer.session.add(artist)
|
||||
importer.session.flush()
|
||||
sc = import_root / "Linker" / "_post.json"
|
||||
sc.parent.mkdir(parents=True, exist_ok=True)
|
||||
sc.write_text(json.dumps({
|
||||
"category": "patreon", "id": 999, "title": "Film",
|
||||
"url": "https://patreon.com/posts/999",
|
||||
"content": '<p>Get it: <a href="https://mega.nz/file/AbC#key123">Mega</a></p>',
|
||||
}))
|
||||
assert importer.upsert_post_record(sc, artist=artist) is True
|
||||
importer.session.expire_all() # re-read so the server_default status loads
|
||||
links = importer.session.execute(select(ExternalLink)).scalars().all()
|
||||
assert len(links) == 1
|
||||
assert links[0].host == "mega"
|
||||
assert links[0].url == "https://mega.nz/file/AbC#key123"
|
||||
assert links[0].label == "Mega"
|
||||
assert links[0].status == "pending"
|
||||
|
||||
|
||||
def test_external_links_not_duplicated_on_reimport(importer, import_layout):
|
||||
"""Re-importing the same body keeps ONE external_link row (insert-missing)."""
|
||||
import_root, _ = import_layout
|
||||
artist = Artist(name="Linker2", slug="linker2")
|
||||
importer.session.add(artist)
|
||||
importer.session.flush()
|
||||
sc = import_root / "Linker2" / "_post.json"
|
||||
sc.parent.mkdir(parents=True, exist_ok=True)
|
||||
sc.write_text(json.dumps({
|
||||
"category": "patreon", "id": 1000, "title": "Film",
|
||||
"url": "https://patreon.com/posts/1000",
|
||||
"content": '<a href="https://pixeldrain.com/u/xyz">px</a>',
|
||||
}))
|
||||
assert importer.upsert_post_record(sc, artist=artist) is True
|
||||
assert importer.upsert_post_record(sc, artist=artist) is True
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(ExternalLink)
|
||||
).scalar_one() == 1
|
||||
|
||||
Reference in New Issue
Block a user