bvandeusenandClaude Opus 5.5
157f2803c0
CI and images / lint (push) Successful in 1s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 28s
CI and images / backend-lint-and-test (push) Successful in 35s
CI and images / integration (push) Successful in 2m25s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 2m3s
CI and images / smoke-web (push) Successful in 55s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
- An archive whose images were extracted was kept at its download path as well as in the attachments store. Both download paths now remove the downloaded copy. - A downloaded file the import filters reject (too small, transparent, single colour, unreadable) was left on disk with no record. A later walk found it, imported it again and rejected it again, every time. Rejected files are now removed; the seen ledger keeps them from being downloaded again. - Both paths share one policy, Importer.discard_after_attach, which never removes a file an ImageRecord points at. - An external-link fetch that fails, or whose worker was killed, left partial files (megatools' .megatmp.*) in the link's dir, and a killed fetch left its link 'downloading' for good. A fetch now clears unrecorded leftovers before starting and after failing. The sweep returns a link to retry once it has been 'downloading' past the task's hard limit, using a new external_link.started_at column (migration 0117). - The orphaned temp-file sweep now recognises .megatmp.* as well as .part and .partial. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
98 lines
3.9 KiB
Python
98 lines
3.9 KiB
Python
"""ExternalLink — an off-platform file-host link found in a post body.
|
|
|
|
Creators host the actual files (films, packs) on mega.nz / Google Drive /
|
|
MediaFire / Dropbox / Pixeldrain and drop the link in the post text. This row
|
|
is the record that the link existed (so nothing is silently dropped), the
|
|
dedup + dead-letter ledger for fetching it, and the driver the download worker
|
|
walks. `url` keeps the FULL link including the `#fragment` (mega's decryption
|
|
key) — truncating it makes the file undownloadable.
|
|
|
|
status lifecycle: pending → downloading → downloaded | failed | dead
|
|
(too many attempts) | skipped (host disabled). `attachment_id` links the
|
|
captured file once a download lands (SET NULL so deleting the attachment
|
|
doesn't delete the link record).
|
|
"""
|
|
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import (
|
|
CheckConstraint,
|
|
DateTime,
|
|
Float,
|
|
ForeignKey,
|
|
Index,
|
|
Integer,
|
|
String,
|
|
Text,
|
|
func,
|
|
text,
|
|
)
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from .base import Base
|
|
|
|
# Kept in sync with link_extract.SUPPORTED_HOSTS and the CHECK in migration 0049.
|
|
HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
|
|
STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead")
|
|
|
|
|
|
class ExternalLink(Base):
|
|
__tablename__ = "external_link"
|
|
__table_args__ = (
|
|
# alembic 0028 enum CHECKs. Rule 36 territory: a new host or status value
|
|
# needs its constraint swapped in the same migration (#3275).
|
|
CheckConstraint(
|
|
"host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')",
|
|
# Bare name: Base.metadata's naming convention prepends
|
|
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
|
# alembic 0088, which renames the four constraints that shipped
|
|
# that way (#3275).
|
|
name="host",
|
|
),
|
|
CheckConstraint(
|
|
"status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')",
|
|
name="status",
|
|
),
|
|
# One row per (post, url). The full url (incl. #fragment) is the identity
|
|
# — the same file linked twice in a post collapses to one row.
|
|
Index("uq_external_link_post_url", "post_id", "url", unique=True),
|
|
Index("ix_external_link_status", "status"),
|
|
# Unindexed FK (#3300).
|
|
Index("ix_external_link_attachment_id", "attachment_id"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
# No index=True: uq_external_link_post_url (post_id, url) already leads
|
|
# with post_id (#3301).
|
|
post_id: Mapped[int] = mapped_column(
|
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
artist_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
|
)
|
|
host: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
label: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
status: Mapped[str] = mapped_column(
|
|
String(16), nullable=False, server_default="pending"
|
|
)
|
|
attempts: Mapped[int] = mapped_column(
|
|
Integer, nullable=False, server_default=text("0")
|
|
)
|
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
attachment_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
|
|
)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
)
|
|
# When the current fetch claimed the link ('downloading'). The sweep uses it
|
|
# to recover a link whose worker died mid-fetch.
|
|
started_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True), nullable=True
|
|
)
|
|
completed_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True), nullable=True
|
|
)
|
|
duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
|