bvandeusenandClaude Opus 5.5
5259caa846
CI and images / lint (push) Successful in 1s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 20s
CI and images / extension-test (push) Successful in 23s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m20s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The daily recovery reset every dead link to a fresh three-attempt budget, so a link that can never succeed (an archive too big to import inside the fetch's hard limit, a file made private) was downloaded up to three times a day, every day. Recovery now gives a dead link one more attempt once it has been dead for 2**recoveries days (1, 2, 4, 8, 16), and stops after five. A recovered link that fails again goes straight back to dead. Retention prunes only links recovery has given up on, 30 days after their last attempt. A manual refetch of a post restores the full budgets. Migration 0118 adds external_link.recoveries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
104 lines
4.2 KiB
Python
104 lines
4.2 KiB
Python
"""ExternalLink — an off-platform file-host link found in a post body.
|
|
|
|
Creators host the actual files (films, packs) on mega.nz / Google Drive /
|
|
MediaFire / Dropbox / Pixeldrain and drop the link in the post text. This row
|
|
is the record that the link existed (so nothing is silently dropped), the
|
|
dedup + dead-letter ledger for fetching it, and the driver the download worker
|
|
walks. `url` keeps the FULL link including the `#fragment` (mega's decryption
|
|
key) — truncating it makes the file undownloadable.
|
|
|
|
status lifecycle: pending → downloading → downloaded | failed | dead
|
|
(too many attempts) | skipped (host disabled). `attachment_id` links the
|
|
captured file once a download lands (SET NULL so deleting the attachment
|
|
doesn't delete the link record).
|
|
"""
|
|
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import (
|
|
CheckConstraint,
|
|
DateTime,
|
|
Float,
|
|
ForeignKey,
|
|
Index,
|
|
Integer,
|
|
String,
|
|
Text,
|
|
func,
|
|
text,
|
|
)
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from .base import Base
|
|
|
|
# Kept in sync with link_extract.SUPPORTED_HOSTS and the CHECK in migration 0049.
|
|
HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
|
|
STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead")
|
|
|
|
|
|
class ExternalLink(Base):
|
|
__tablename__ = "external_link"
|
|
__table_args__ = (
|
|
# alembic 0028 enum CHECKs. Rule 36 territory: a new host or status value
|
|
# needs its constraint swapped in the same migration (#3275).
|
|
CheckConstraint(
|
|
"host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')",
|
|
# Bare name: Base.metadata's naming convention prepends
|
|
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
|
# alembic 0088, which renames the four constraints that shipped
|
|
# that way (#3275).
|
|
name="host",
|
|
),
|
|
CheckConstraint(
|
|
"status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')",
|
|
name="status",
|
|
),
|
|
# One row per (post, url). The full url (incl. #fragment) is the identity
|
|
# — the same file linked twice in a post collapses to one row.
|
|
Index("uq_external_link_post_url", "post_id", "url", unique=True),
|
|
Index("ix_external_link_status", "status"),
|
|
# Unindexed FK (#3300).
|
|
Index("ix_external_link_attachment_id", "attachment_id"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
# No index=True: uq_external_link_post_url (post_id, url) already leads
|
|
# with post_id (#3301).
|
|
post_id: Mapped[int] = mapped_column(
|
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
artist_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
|
)
|
|
host: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
label: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
status: Mapped[str] = mapped_column(
|
|
String(16), nullable=False, server_default="pending"
|
|
)
|
|
attempts: Mapped[int] = mapped_column(
|
|
Integer, nullable=False, server_default=text("0")
|
|
)
|
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
# Times the daily recovery has given this link another attempt after it
|
|
# went dead. Recovery stops at a limit, so a link that can never succeed
|
|
# isn't downloaded forever.
|
|
recoveries: Mapped[int] = mapped_column(
|
|
Integer, nullable=False, server_default=text("0")
|
|
)
|
|
attachment_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
|
|
)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
)
|
|
# When the current fetch claimed the link ('downloading'). The sweep uses it
|
|
# to recover a link whose worker died mid-fetch.
|
|
started_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True), nullable=True
|
|
)
|
|
completed_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True), nullable=True
|
|
)
|
|
duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
|