FabledCurator/tests/test_download_source_task.py

"""Tests for the FC-3c download_source Celery task wrapper.

Covers registration/routing (smoke) plus the soft-time-limit salvage
path (audit 2026-06-03, Anduo #39912): a SoftTimeLimitExceeded must not
leave the DownloadEvent stranded empty for the recovery sweep.
"""

from datetime import UTC, datetime

import pytest
from sqlalchemy import select

# Side-effect import: the @celery.task decorator on download_source fires
# at module import time and registers the task with the global instance.
import backend.app.tasks.download  # noqa: F401
from backend.app.celery_app import celery


def test_download_source_is_registered():
    assert "backend.app.tasks.download.download_source" in celery.tasks


def test_download_source_routes_to_download_queue():
    routes = celery.conf.task_routes
    assert "backend.app.tasks.download.*" in routes
    assert routes["backend.app.tasks.download.*"]["queue"] == "download"


def test_timeout_ladder_keeps_subprocess_budgets_under_soft_limit():
    """Regression guard for Anduo #39912: every gallery-dl subprocess
    budget MUST sit below download_source's Celery soft limit so
    subprocess.run raises its own TimeoutExpired (which captures partial
    logs + finalizes the event) BEFORE Celery's SoftTimeLimitExceeded
    preempts it. soft must in turn sit below the hard SIGKILL cap."""
    from backend.app.services.gallery_dl import (
        _DEFAULT_GDL_TIMEOUT_SECONDS,
        BACKFILL_CHUNK_SECONDS,
    )
    from backend.app.tasks.download import (
        DOWNLOAD_HARD_TIME_LIMIT,
        DOWNLOAD_SOFT_TIME_LIMIT,
    )

    assert _DEFAULT_GDL_TIMEOUT_SECONDS < DOWNLOAD_SOFT_TIME_LIMIT
    assert BACKFILL_CHUNK_SECONDS < DOWNLOAD_SOFT_TIME_LIMIT
    assert DOWNLOAD_SOFT_TIME_LIMIT < DOWNLOAD_HARD_TIME_LIMIT


def test_the_native_chunk_leaves_room_to_tear_down_before_the_soft_limit():
    """The native walk sizes itself against CHUNK_TOTAL_SECONDS, walk plus
    phase 3; that total has to leave the task time to finalize its event."""
    from backend.app.services.ingest_core import CHUNK_TOTAL_SECONDS
    from backend.app.tasks.download import DOWNLOAD_SOFT_TIME_LIMIT

    assert CHUNK_TOTAL_SECONDS <= DOWNLOAD_SOFT_TIME_LIMIT - 120


def test_phase_3_stops_importing_with_room_to_finalize():
    """Past its deadline phase 3 starts no import; what remains after it (the
    import in hand, post records, the event) has to fit before the soft limit."""
    from backend.app.services.ingest_core import PHASE3_DEADLINE_SECONDS
    from backend.app.tasks.download import DOWNLOAD_SOFT_TIME_LIMIT

    assert PHASE3_DEADLINE_SECONDS <= DOWNLOAD_SOFT_TIME_LIMIT - 120


def test_decorated_limits_match_module_constants():
    """The @celery.task decorator must use the audited constants, not
    drifted literals."""
    from backend.app.tasks.download import (
        DOWNLOAD_HARD_TIME_LIMIT,
        DOWNLOAD_SOFT_TIME_LIMIT,
        download_source,
    )

    assert download_source.soft_time_limit == DOWNLOAD_SOFT_TIME_LIMIT
    assert download_source.time_limit == DOWNLOAD_HARD_TIME_LIMIT


def _seed_running_event(db_sync, *, slug, backfill, failures=0):
    from backend.app.models import Artist, DownloadEvent, Source

    artist = Artist(name=slug, slug=slug)
    db_sync.add(artist)
    db_sync.flush()
    source = Source(
        artist_id=artist.id, platform="patreon",
        url=f"https://patreon.com/{slug}", enabled=True,
        config_overrides={}, backfill_runs_remaining=backfill,
        consecutive_failures=failures,
    )
    db_sync.add(source)
    db_sync.flush()
    ev = DownloadEvent(
        source_id=source.id, status="running",
        started_at=datetime.now(UTC),
    )
    db_sync.add(ev)
    db_sync.flush()
    return source, ev.id


@pytest.mark.integration
@pytest.mark.asyncio
async def test_finalize_soft_limited_flips_event_and_decrements_backfill(db_sync):
    from backend.app.models import DownloadEvent, Source
    from backend.app.tasks.download import _finalize_soft_limited

    source, event_id = _seed_running_event(
        db_sync, slug="anduo", backfill=2, failures=0,
    )

    _finalize_soft_limited(db_sync, source.id)

    status, finished_at, error, meta = db_sync.execute(
        select(
            DownloadEvent.status, DownloadEvent.finished_at,
            DownloadEvent.error, DownloadEvent.metadata_,
        ).where(DownloadEvent.id == event_id)
    ).one()
    assert status == "error"
    assert finished_at is not None
    assert "soft time limit" in (error or "").lower()
    assert meta.get("error_type") == "timeout"
    assert meta.get("soft_time_limited") is True

    backfill, failures, error_type = db_sync.execute(
        select(
            Source.backfill_runs_remaining, Source.consecutive_failures,
            Source.error_type,
        ).where(Source.id == source.id)
    ).one()
    assert backfill == 1          # 2 -> 1, source self-heals toward tick mode
    assert failures == 1          # 0 -> 1, mirrors phase-3 source-health write
    assert error_type == "timeout"


@pytest.mark.integration
@pytest.mark.asyncio
async def test_finalize_soft_limited_is_noop_without_running_event(db_sync):
    """A benign late soft-limit (phase 3 already committed → no running
    event) must not touch source health or backfill budget."""
    from backend.app.models import DownloadEvent, Source
    from backend.app.tasks.download import _finalize_soft_limited

    source, event_id = _seed_running_event(
        db_sync, slug="noevent", backfill=3, failures=0,
    )
    # Simulate phase 3 having already finalized the event.
    ev = db_sync.get(DownloadEvent, event_id)
    ev.status = "ok"
    db_sync.flush()

    _finalize_soft_limited(db_sync, source.id)

    backfill, failures, error_type = db_sync.execute(
        select(
            Source.backfill_runs_remaining, Source.consecutive_failures,
            Source.error_type,
        ).where(Source.id == source.id)
    ).one()
    assert backfill == 3          # untouched
    assert failures == 0          # untouched
    assert error_type is None


@pytest.mark.integration
@pytest.mark.asyncio
async def test_download_source_catches_soft_limit_and_salvages_event(
    db_sync, monkeypatch,
):
    """End-to-end wiring: when the inner run raises SoftTimeLimitExceeded,
    download_source's handler must flip the in-flight event to error
    instead of letting it strand. Uses eager mode + a stubbed asyncio.run
    so no real gallery-dl subprocess is spawned."""
    from celery.exceptions import SoftTimeLimitExceeded

    import backend.app.tasks.download as dl
    from backend.app.models import DownloadEvent

    monkeypatch.setattr(celery.conf, "task_always_eager", True)
    monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
    # This test exercises the soft-limit salvage path, not the per-platform
    # serialization. Neutralize the Redis lock so it can't defer/recurse under
    # eager mode (and so it doesn't contend with test_platform_lock).
    monkeypatch.setattr(
        "backend.app.services.platform_lock.platform_lock", lambda *a, **k: None,
    )

    source, event_id = _seed_running_event(
        db_sync, slug="anduowire", backfill=1, failures=0,
    )
    # The task opens a fresh session via _sync_session_factory(); commit
    # so that session can see the seeded running event.
    db_sync.commit()

    def _raise(coro=None, *a, **k):
        # Close the un-awaited coroutine so pytest output stays pristine.
        if coro is not None and hasattr(coro, "close"):
            coro.close()
        raise SoftTimeLimitExceeded("simulated soft limit")

    monkeypatch.setattr(dl.asyncio, "run", _raise)

    with pytest.raises(SoftTimeLimitExceeded):
        dl.download_source.delay(source.id).get(propagate=True)

    status = db_sync.execute(
        select(DownloadEvent.status).where(DownloadEvent.id == event_id)
    ).scalar_one()
    assert status == "error"