bvandeusen
d96918d777
Capture off-platform links (mega/gdrive/mediafire/dropbox/pixeldrain) embedded in post bodies so they're never silently dropped, and surface them in the post view. The download worker (Phase 4) walks these rows. - link_extract.py: pure extractor — <a href> + bare URLs, unwraps Patreon redirect shims, PRESERVES the full url incl. #fragment (mega's key), dedups. Reusable by every platform (runs off Post.description). - external_link model + migration 0049: post_id/artist_id/host/url/label/status /attempts/last_error/attachment_id/timing; CHECK whitelists (full enum incl. worker statuses up front) + (post_id,url) unique. - importer._sync_external_links: insert-missing on both import paths (_apply_sidecar + upsert_post_record) so a re-import never resets a link's status; runs for all platforms. - post_feed_service.get_post: returns external_links (detail-only). - PostCard: renders the links (host chip + label + status) once expanded. - tests: extractor (5 hosts, fragment, shim unwrap, dedup), importer (record + no-dup on reimport), serializer. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
91 lines
3.3 KiB
Python
91 lines
3.3 KiB
Python
"""external_link table — off-platform file-host links found in post bodies
|
|
|
|
Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox /
|
|
Pixeldrain and link them in the post text. This table records each such link
|
|
(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger
|
|
the download worker (a later slice) walks. `url` keeps the FULL link including
|
|
the `#fragment` — mega.nz's decryption key lives there; truncating it makes the
|
|
file undownloadable.
|
|
|
|
CHECK whitelists for host + status include the full enum up front (incl. the
|
|
download-worker statuses) so the worker slice needs no constraint migration.
|
|
|
|
Revision ID: 0049
|
|
Revises: 0048
|
|
Create Date: 2026-06-14
|
|
"""
|
|
from typing import Sequence, Union
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
revision: str = "0049"
|
|
down_revision: Union[str, None] = "0048"
|
|
branch_labels: Union[str, Sequence[str], None] = None
|
|
depends_on: Union[str, Sequence[str], None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"external_link",
|
|
sa.Column("id", sa.Integer(), primary_key=True),
|
|
sa.Column(
|
|
"post_id", sa.Integer(),
|
|
sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False,
|
|
),
|
|
sa.Column(
|
|
"artist_id", sa.Integer(),
|
|
sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True,
|
|
),
|
|
sa.Column("host", sa.String(length=16), nullable=False),
|
|
sa.Column("url", sa.Text(), nullable=False),
|
|
sa.Column("label", sa.Text(), nullable=True),
|
|
sa.Column(
|
|
"status", sa.String(length=16), nullable=False,
|
|
server_default="pending",
|
|
),
|
|
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("last_error", sa.Text(), nullable=True),
|
|
sa.Column(
|
|
"attachment_id", sa.Integer(),
|
|
sa.ForeignKey("post_attachment.id", ondelete="SET NULL"),
|
|
nullable=True,
|
|
),
|
|
sa.Column(
|
|
"created_at", sa.DateTime(timezone=True), nullable=False,
|
|
server_default=sa.func.now(),
|
|
),
|
|
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("duration_seconds", sa.Float(), nullable=True),
|
|
sa.CheckConstraint(
|
|
"host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')",
|
|
name="ck_external_link_host",
|
|
),
|
|
sa.CheckConstraint(
|
|
"status IN ('pending','downloading','downloaded','failed',"
|
|
"'skipped','dead')",
|
|
name="ck_external_link_status",
|
|
),
|
|
)
|
|
op.create_index(
|
|
"ix_external_link_post_id", "external_link", ["post_id"],
|
|
)
|
|
op.create_index(
|
|
"ix_external_link_artist_id", "external_link", ["artist_id"],
|
|
)
|
|
op.create_index(
|
|
"ix_external_link_status", "external_link", ["status"],
|
|
)
|
|
op.create_index(
|
|
"uq_external_link_post_url", "external_link", ["post_id", "url"],
|
|
unique=True,
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_index("uq_external_link_post_url", table_name="external_link")
|
|
op.drop_index("ix_external_link_status", table_name="external_link")
|
|
op.drop_index("ix_external_link_artist_id", table_name="external_link")
|
|
op.drop_index("ix_external_link_post_id", table_name="external_link")
|
|
op.drop_table("external_link")
|