feat(posts): extract + record external file-host links (Phase 3)
Capture off-platform links (mega/gdrive/mediafire/dropbox/pixeldrain) embedded in post bodies so they're never silently dropped, and surface them in the post view. The download worker (Phase 4) walks these rows. - link_extract.py: pure extractor — <a href> + bare URLs, unwraps Patreon redirect shims, PRESERVES the full url incl. #fragment (mega's key), dedups. Reusable by every platform (runs off Post.description). - external_link model + migration 0049: post_id/artist_id/host/url/label/status /attempts/last_error/attachment_id/timing; CHECK whitelists (full enum incl. worker statuses up front) + (post_id,url) unique. - importer._sync_external_links: insert-missing on both import paths (_apply_sidecar + upsert_post_record) so a re-import never resets a link's status; runs for all platforms. - post_feed_service.get_post: returns external_links (detail-only). - PostCard: renders the links (host chip + label + status) once expanded. - tests: extractor (5 hosts, fragment, shim unwrap, dedup), importer (record + no-dup on reimport), serializer. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
Artist,
|
||||
ExternalLink,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
@@ -39,6 +40,7 @@ from ..utils.paths import (
|
||||
)
|
||||
from ..utils.phash import compute_phash, find_similar
|
||||
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||
from .link_extract import extract_external_links
|
||||
from ..utils.slug import slugify
|
||||
from .archive_extractor import extract_archive, is_archive
|
||||
from .attachment_store import AttachmentStore
|
||||
@@ -305,6 +307,32 @@ class Importer:
|
||||
),
|
||||
)
|
||||
|
||||
def _sync_external_links(self, post: Post) -> None:
|
||||
"""Record off-platform file-host links (mega/gdrive/mediafire/dropbox/
|
||||
pixeldrain) found in the post body, so they're never silently dropped
|
||||
and the download worker can fetch them later. Shared by every platform's
|
||||
import (runs off Post.description). INSERT-MISSING only — an existing row
|
||||
keeps its status/attempts (a re-import must not reset a link already
|
||||
downloaded or dead-lettered). Identity is (post_id, url); the full url
|
||||
incl. #fragment is preserved by the extractor."""
|
||||
links = extract_external_links(post.description)
|
||||
if not links:
|
||||
return
|
||||
self.session.flush() # ensure post.id is assigned before we reference it
|
||||
existing = set(self.session.execute(
|
||||
select(ExternalLink.url).where(ExternalLink.post_id == post.id)
|
||||
).scalars().all())
|
||||
for link in links:
|
||||
if link.url in existing:
|
||||
continue
|
||||
self.session.add(ExternalLink(
|
||||
post_id=post.id,
|
||||
artist_id=post.artist_id,
|
||||
host=link.host,
|
||||
url=link.url,
|
||||
label=link.label,
|
||||
))
|
||||
|
||||
def import_one(self, source: Path) -> ImportResult:
|
||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||
media members (one Post via the archive-adjacent sidecar) and
|
||||
@@ -767,6 +795,7 @@ class Importer:
|
||||
if sd.attachment_count is not None:
|
||||
post.attachment_count = sd.attachment_count
|
||||
post.raw_metadata = sd.raw
|
||||
self._sync_external_links(post)
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
@@ -1042,6 +1071,7 @@ class Importer:
|
||||
if sd.attachment_count is not None:
|
||||
post.attachment_count = sd.attachment_count
|
||||
post.raw_metadata = sd.raw
|
||||
self._sync_external_links(post)
|
||||
|
||||
# Race-safe (image_record_id, post_id) upsert — mirrors the
|
||||
# _find_or_create_source/post savepoint pattern. The plain
|
||||
|
||||
Reference in New Issue
Block a user