bvandeusen
d96918d777
Capture off-platform links (mega/gdrive/mediafire/dropbox/pixeldrain) embedded in post bodies so they're never silently dropped, and surface them in the post view. The download worker (Phase 4) walks these rows. - link_extract.py: pure extractor — <a href> + bare URLs, unwraps Patreon redirect shims, PRESERVES the full url incl. #fragment (mega's key), dedups. Reusable by every platform (runs off Post.description). - external_link model + migration 0049: post_id/artist_id/host/url/label/status /attempts/last_error/attachment_id/timing; CHECK whitelists (full enum incl. worker statuses up front) + (post_id,url) unique. - importer._sync_external_links: insert-missing on both import paths (_apply_sidecar + upsert_post_record) so a re-import never resets a link's status; runs for all platforms. - post_feed_service.get_post: returns external_links (detail-only). - PostCard: renders the links (host chip + label + status) once expanded. - tests: extractor (5 hosts, fragment, shim unwrap, dedup), importer (record + no-dup on reimport), serializer. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
74 lines
2.8 KiB
Python
74 lines
2.8 KiB
Python
"""ExternalLink — an off-platform file-host link found in a post body.
|
|
|
|
Creators host the actual files (films, packs) on mega.nz / Google Drive /
|
|
MediaFire / Dropbox / Pixeldrain and drop the link in the post text. This row
|
|
is the record that the link existed (so nothing is silently dropped), the
|
|
dedup + dead-letter ledger for fetching it, and the driver the download worker
|
|
walks. `url` keeps the FULL link including the `#fragment` (mega's decryption
|
|
key) — truncating it makes the file undownloadable.
|
|
|
|
status lifecycle: pending → downloading → downloaded | failed | dead
|
|
(too many attempts) | skipped (host disabled). `attachment_id` links the
|
|
captured file once a download lands (SET NULL so deleting the attachment
|
|
doesn't delete the link record).
|
|
"""
|
|
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import (
|
|
DateTime,
|
|
Float,
|
|
ForeignKey,
|
|
Index,
|
|
Integer,
|
|
String,
|
|
Text,
|
|
func,
|
|
text,
|
|
)
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from .base import Base
|
|
|
|
# Kept in sync with link_extract.SUPPORTED_HOSTS and the CHECK in migration 0049.
|
|
HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
|
|
STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead")
|
|
|
|
|
|
class ExternalLink(Base):
|
|
__tablename__ = "external_link"
|
|
__table_args__ = (
|
|
# One row per (post, url). The full url (incl. #fragment) is the identity
|
|
# — the same file linked twice in a post collapses to one row.
|
|
Index("uq_external_link_post_url", "post_id", "url", unique=True),
|
|
Index("ix_external_link_status", "status"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
post_id: Mapped[int] = mapped_column(
|
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
|
)
|
|
artist_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
|
)
|
|
host: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
label: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
status: Mapped[str] = mapped_column(
|
|
String(16), nullable=False, server_default="pending"
|
|
)
|
|
attempts: Mapped[int] = mapped_column(
|
|
Integer, nullable=False, server_default=text("0")
|
|
)
|
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
attachment_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
|
|
)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
)
|
|
completed_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True), nullable=True
|
|
)
|
|
duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
|