feat(posts): extract + record external file-host links (Phase 3)
Capture off-platform links (mega/gdrive/mediafire/dropbox/pixeldrain) embedded in post bodies so they're never silently dropped, and surface them in the post view. The download worker (Phase 4) walks these rows. - link_extract.py: pure extractor — <a href> + bare URLs, unwraps Patreon redirect shims, PRESERVES the full url incl. #fragment (mega's key), dedups. Reusable by every platform (runs off Post.description). - external_link model + migration 0049: post_id/artist_id/host/url/label/status /attempts/last_error/attachment_id/timing; CHECK whitelists (full enum incl. worker statuses up front) + (post_id,url) unique. - importer._sync_external_links: insert-missing on both import paths (_apply_sidecar + upsert_post_record) so a re-import never resets a link's status; runs for all platforms. - post_feed_service.get_post: returns external_links (detail-only). - PostCard: renders the links (host chip + label + status) once expanded. - tests: extractor (5 hosts, fragment, shim unwrap, dedup), importer (record + no-dup on reimport), serializer. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
"""external_link table — off-platform file-host links found in post bodies
|
||||
|
||||
Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox /
|
||||
Pixeldrain and link them in the post text. This table records each such link
|
||||
(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger
|
||||
the download worker (a later slice) walks. `url` keeps the FULL link including
|
||||
the `#fragment` — mega.nz's decryption key lives there; truncating it makes the
|
||||
file undownloadable.
|
||||
|
||||
CHECK whitelists for host + status include the full enum up front (incl. the
|
||||
download-worker statuses) so the worker slice needs no constraint migration.
|
||||
|
||||
Revision ID: 0049
|
||||
Revises: 0048
|
||||
Create Date: 2026-06-14
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0049"
|
||||
down_revision: Union[str, None] = "0048"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"external_link",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"post_id", sa.Integer(),
|
||||
sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"artist_id", sa.Integer(),
|
||||
sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True,
|
||||
),
|
||||
sa.Column("host", sa.String(length=16), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("label", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"status", sa.String(length=16), nullable=False,
|
||||
server_default="pending",
|
||||
),
|
||||
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"attachment_id", sa.Integer(),
|
||||
sa.ForeignKey("post_attachment.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("duration_seconds", sa.Float(), nullable=True),
|
||||
sa.CheckConstraint(
|
||||
"host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')",
|
||||
name="ck_external_link_host",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('pending','downloading','downloaded','failed',"
|
||||
"'skipped','dead')",
|
||||
name="ck_external_link_status",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_external_link_post_id", "external_link", ["post_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_external_link_artist_id", "external_link", ["artist_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_external_link_status", "external_link", ["status"],
|
||||
)
|
||||
op.create_index(
|
||||
"uq_external_link_post_url", "external_link", ["post_id", "url"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("uq_external_link_post_url", table_name="external_link")
|
||||
op.drop_index("ix_external_link_status", table_name="external_link")
|
||||
op.drop_index("ix_external_link_artist_id", table_name="external_link")
|
||||
op.drop_index("ix_external_link_post_id", table_name="external_link")
|
||||
op.drop_table("external_link")
|
||||
Reference in New Issue
Block a user