feat(external): download worker for file-host links (Phase 4b)

tasks/external.py drives the external_link ledger: - fetch_external_link(link_id): atomic claim (pending/failed→downloading, so a duplicate enqueue no-ops), per-host Redis serialize lock (#720 pattern; requeue-with-countdown if busy), fetch via external_fetch into the artist library tree, then route each file through importer.attach_in_place via a synthesized sidecar so it links to the SAME post (archive→ImageRecords, else→PostAttachment; on-disk original removed for captured files, art stays); thumbnail+ML enqueue for new images; status downloaded | failed | dead with attempts/last_error/completed_at/duration. - sweep_external_links(): enqueue a bounded batch of actionable links. - recover_external_links() + prune_external_links(): recovery + retention (#89). - per-host enable read via getattr (forward-compatible; Settings UI adds the columns in 4d — defaults on, rule #26). Wiring: celery include + route (download lane) + beat (sweep 10m, recover + prune daily); download_service phase 3 enqueues a sweep after recording links. Integration tests: download+attach, failure, dead-letter, non-claimable, sweep. mega still needs the MEGAcmd binary in the runtime image (Phase 4c). Refs #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-14 13:44:07 -04:00
parent 13253b18d1
commit 96e984cded
4 changed files with 455 additions and 0 deletions
@@ -0,0 +1,292 @@
+"""Celery worker for off-platform file-host downloads (external_link ledger).
+
+Walks the external_link table and fetches each pending/retryable link via
+external_fetch, then routes the downloaded file(s) through the existing importer
+(attach_in_place) so an archive becomes ImageRecords and any other file a
+PostAttachment — linked to the SAME post the link came from (via a synthesized
+sidecar). Long-running-task hygiene (rule #89): per-fetch wall-clock timeout,
+attempt tracking + dead-letter, a recovery sweep, and retention of dead rows.
+
+Concurrency: a per-host Redis lock serializes fetches to one-per-host (mega /
+gdrive ban-avoidance), and an atomic status claim (pending/failed →
+downloading) stops two workers grabbing the same link.
+
+Files land in the artist's library tree (so an attached art image stays in
+place, mirroring the gallery-dl download path); a captured archive/attachment is
+copied into its store and the on-disk original removed.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import time
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import redis
+from sqlalchemy import delete, select, update
+
+from ..celery_app import celery
+from ..config import get_config
+from ..models import Artist, ExternalLink, ImportSettings, Post, Source
+from ..services.external_fetch import fetch_external
+from ..services.importer import Importer
+from ..services.thumbnailer import Thumbnailer
+from ._sync_engine import sync_session_factory as _sync_session_factory
+
+log = logging.getLogger(__name__)
+
+IMAGES_ROOT = Path("/images")
+
+# After this many failed attempts a link is dead-lettered (skipped by routine
+# sweeps; an operator recovery still re-attempts). Mirrors the ingester ledger.
+DEAD_LETTER_THRESHOLD = 3
+# Per-fetch wall-clock budget — films/packs are large; the celery time_limit is
+# the backstop above this.
+_FETCH_TIMEOUT = 3000.0
+# Links enqueued per sweep — bounds the burst when a big backfill records many.
+_SWEEP_BATCH = 50
+# Dead rows older than this are pruned (retention).
+_RETENTION_DAYS = 30
+# Per-host serialize lock.
+_LOCK_PREFIX = "fc:extdl_lock:"
+_LOCK_TTL = 3600
+_SERIALIZE_COUNTDOWN = 120
+_MAX_SERIALIZE_WAITS = 30
+
+_redis_client: redis.Redis | None = None
+
+
+def _redis() -> redis.Redis:
+    global _redis_client
+    if _redis_client is None:
+        _redis_client = redis.from_url(get_config().celery_broker_url)
+    return _redis_client
+
+
+def _host_enabled(settings: ImportSettings, host: str) -> bool:
+    """Per-host enable flag — defaults True (rule #26: works out of the box).
+    The Settings UI slice adds real columns; getattr keeps this forward-
+    compatible without a migration in this slice."""
+    return bool(getattr(settings, f"extdl_{host}_enabled", True))
+
+
+def _write_link_sidecar(file: Path, post: Post, platform: str, artist: Artist) -> None:
+    """Sidecar next to a fetched file so the importer links it to `post`
+    (category+id resolve the source+post; matches gallery-dl's sidecar shape)."""
+    data = {
+        "category": platform,
+        "id": post.external_post_id,
+        "url": post.post_url,
+        "title": post.post_title,
+        "artist": artist.name,
+    }
+    Path(str(file) + ".json").write_text(json.dumps(data))
+
+
+@celery.task(
+    name="backend.app.tasks.external.fetch_external_link",
+    bind=True,
+    soft_time_limit=3300,
+    time_limit=3600,
+)
+def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
+    """Fetch one external_link and route its file(s) through the importer."""
+    SessionLocal = _sync_session_factory()
+
+    # Atomic claim: only an actionable (pending/failed) row transitions to
+    # downloading, so a duplicate enqueue (sweep + post-download hook) no-ops.
+    with SessionLocal() as session:
+        claimed = session.execute(
+            update(ExternalLink)
+            .where(
+                ExternalLink.id == link_id,
+                ExternalLink.status.in_(("pending", "failed")),
+            )
+            .values(status="downloading")
+            .returning(ExternalLink.id)
+        ).first()
+        session.commit()
+        if claimed is None:
+            return {"link_id": link_id, "skipped": "not claimable"}
+
+        link = session.get(ExternalLink, link_id)
+        post = session.get(Post, link.post_id) if link.post_id else None
+        if post is None:
+            link.status = "dead"
+            link.last_error = "post missing"
+            link.completed_at = datetime.now(UTC)
+            session.commit()
+            return {"link_id": link_id, "error": "post missing"}
+        artist = session.get(Artist, post.artist_id)
+        source = session.get(Source, post.source_id) if post.source_id else None
+        platform = source.platform if source is not None else "patreon"
+        settings = ImportSettings.load_sync(session)
+
+        if not _host_enabled(settings, link.host):
+            link.status = "skipped"
+            link.last_error = f"{link.host} disabled"
+            link.completed_at = datetime.now(UTC)
+            session.commit()
+            return {"link_id": link_id, "skipped": "host disabled"}
+
+        host, url, attempts = link.host, link.url, link.attempts
+
+    # Per-host serialize: one fetch per host at a time. If busy, requeue.
+    lock = None
+    try:
+        lock = _redis().lock(f"{_LOCK_PREFIX}{host}", timeout=_LOCK_TTL, blocking=False)
+        got = bool(lock.acquire(blocking=False))
+    except redis.RedisError as exc:
+        log.warning("extdl lock unavailable for %s: %s — running uncapped", host, exc)
+        lock, got = None, True
+    if not got:
+        # Release the claim back to pending so a later run can pick it up, then
+        # requeue with a countdown (bounded, like download_source).
+        with SessionLocal() as session:
+            session.execute(
+                update(ExternalLink).where(ExternalLink.id == link_id)
+                .values(status="pending")
+            )
+            session.commit()
+        if _serialize_waits < _MAX_SERIALIZE_WAITS:
+            fetch_external_link.apply_async(
+                (link_id,), {"_serialize_waits": _serialize_waits + 1},
+                countdown=_SERIALIZE_COUNTDOWN,
+            )
+        return {"link_id": link_id, "requeued": "host busy"}
+
+    started = time.monotonic()
+    post_dir = IMAGES_ROOT / artist.slug / platform / "external" / str(post.external_post_id)
+    try:
+        result = fetch_external(host, url, post_dir, timeout=_FETCH_TIMEOUT)
+        with SessionLocal() as session:
+            link = session.get(ExternalLink, link_id)
+            if not result.ok:
+                _record_failure(link, attempts, result.error or "fetch failed")
+                session.commit()
+                return {"link_id": link_id, "error": result.error}
+
+            # Re-load post/artist/source attached to THIS session before handing
+            # them to the importer (the claim block's instances are detached).
+            post = session.get(Post, link.post_id)
+            artist = session.get(Artist, post.artist_id)
+            source = session.get(Source, post.source_id) if post.source_id else None
+            image_ids = _route_files(session, result.files, post, platform, artist, source)
+            link.status = "downloaded"
+            link.last_error = None
+            link.completed_at = datetime.now(UTC)
+            link.duration_seconds = time.monotonic() - started
+            session.commit()
+
+        # Thumbnails + ML for any newly-attached images (mirrors the download
+        # path). Lazy import to dodge a task-module import cycle.
+        if image_ids:
+            from .ml import tag_and_embed
+            from .thumbnail import generate_thumbnail
+            for img_id in image_ids:
+                generate_thumbnail.delay(img_id)
+                tag_and_embed.delay(img_id)
+        return {"link_id": link_id, "files": len(result.files), "images": len(image_ids)}
+    except Exception as exc:  # never leave a link stuck in 'downloading'
+        log.exception("external fetch task failed for link %s", link_id)
+        with SessionLocal() as session:
+            link = session.get(ExternalLink, link_id)
+            if link is not None and link.status == "downloading":
+                _record_failure(link, attempts, f"{type(exc).__name__}: {exc}")
+                session.commit()
+        raise
+    finally:
+        if lock is not None:
+            try:
+                lock.release()
+            except Exception:  # noqa: BLE001 — TTL may have already freed it
+                pass
+
+
+def _record_failure(link: ExternalLink, attempts: int, error: str) -> None:
+    nxt = attempts + 1
+    link.attempts = nxt
+    link.last_error = error[:1000]
+    link.status = "dead" if nxt >= DEAD_LETTER_THRESHOLD else "failed"
+    link.completed_at = datetime.now(UTC)
+
+
+def _route_files(session, files, post, platform, artist, source) -> list[int]:
+    """Import each fetched file via the existing pipeline (in the artist tree, so
+    art stays in place); return new image ids for thumb/ML enqueue. A captured
+    archive/attachment is copied into its store, so its on-disk original is
+    removed (mirrors download_service); art images stay."""
+    settings = ImportSettings.load_sync(session)
+    importer = Importer(
+        session=session,
+        images_root=IMAGES_ROOT,
+        import_root=IMAGES_ROOT,
+        thumbnailer=Thumbnailer(images_root=IMAGES_ROOT),
+        settings=settings,
+    )
+    image_ids: list[int] = []
+    for f in files:
+        _write_link_sidecar(f, post, platform, artist)
+        result = importer.attach_in_place(f, artist=artist, source=source)
+        session.commit()
+        if result.status in ("imported", "superseded"):
+            ids = list(getattr(result, "member_image_ids", []) or [])
+            if result.image_id is not None and result.image_id not in ids:
+                ids.append(result.image_id)
+            image_ids.extend(ids)
+        elif result.status in ("attached", "failed"):
+            # Copied into the attachment store (or failed) — drop the on-disk
+            # original so we don't keep two copies / an orphan.
+            f.unlink(missing_ok=True)
+        # The synthesized sidecar has done its job — don't litter the tree.
+        Path(str(f) + ".json").unlink(missing_ok=True)
+    return image_ids
+
+
+@celery.task(name="backend.app.tasks.external.sweep_external_links")
+def sweep_external_links() -> dict:
+    """Enqueue a bounded batch of actionable links (pending, or failed below the
+    dead-letter threshold). Driven by the post-download hook and a beat tick."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        ids = session.execute(
+            select(ExternalLink.id).where(
+                ExternalLink.status.in_(("pending", "failed")),
+                ExternalLink.attempts < DEAD_LETTER_THRESHOLD,
+            ).order_by(ExternalLink.id.asc()).limit(_SWEEP_BATCH)
+        ).scalars().all()
+    for link_id in ids:
+        fetch_external_link.delay(link_id)
+    return {"enqueued": len(ids)}
+
+
+@celery.task(name="backend.app.tasks.external.recover_external_links")
+def recover_external_links() -> dict:
+    """Recovery sweep (rule #89): reset dead links back to retryable so a stuck
+    host outage or a since-fixed bug gets another pass, then enqueue. Bounded."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        session.execute(
+            update(ExternalLink).where(ExternalLink.status == "dead")
+            .values(status="failed", attempts=0, last_error=None)
+        )
+        session.commit()
+    return sweep_external_links()
+
+
+@celery.task(name="backend.app.tasks.external.prune_external_links")
+def prune_external_links() -> dict:
+    """Retention (rule #89): delete long-dead links so the ledger doesn't grow
+    unbounded. Downloaded links are kept (they're the record of what we have)."""
+    cutoff = datetime.now(UTC) - timedelta(days=_RETENTION_DAYS)
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        res = session.execute(
+            delete(ExternalLink).where(
+                ExternalLink.status == "dead",
+                ExternalLink.created_at < cutoff,
+            )
+        )
+        session.commit()
+    return {"pruned": res.rowcount or 0}