cd43439401
Owning the walk lets Stop interrupt a live backfill chunk instead of letting it run to its ~14.5-min time-box. ingest_core.run now polls _backfill_state at each page boundary (a short SELECT, never held across the walk) and bails with PARTIAL when an operator Stop has popped it. Latched on the first observed "running" state so a run invoked without one (unit test / stale call) never self-cancels. Progress is already checkpointed per-page, so a restart resumes from the cursor; Stop clears it for a clean reset. No UI change — the existing Stop button now just takes effect immediately. Tests: _still_running reads the state; a latched run bails PARTIAL at the next boundary when the state disappears (only the pre-cancel post ran). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
673 lines
25 KiB
Python
673 lines
25 KiB
Python
"""PatreonIngester tests (native ingester build step 3).
|
|
|
|
The HTTP client and the media downloader are stubbed (injected seams), so these
|
|
exercise the ingester's own logic — mode behavior, the two-tier skip, cursor
|
|
emission, the budget cutoff, and the Postgres seen-ledger — without network or a
|
|
real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so
|
|
the tier-1 skip and the idempotent mark-seen run against actual rows.
|
|
"""
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from backend.app.models import (
|
|
Artist,
|
|
PatreonFailedMedia,
|
|
PatreonSeenMedia,
|
|
Source,
|
|
)
|
|
from backend.app.services.gallery_dl import ErrorType
|
|
from backend.app.services.patreon_client import (
|
|
MediaItem,
|
|
PatreonAPIError,
|
|
PatreonAuthError,
|
|
PatreonDriftError,
|
|
)
|
|
from backend.app.services.patreon_downloader import MediaOutcome
|
|
from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
# --- fakes ----------------------------------------------------------------
|
|
|
|
|
|
def _media(post_id, n, *, filehash=None, kind="images"):
|
|
# Deterministic 32-hex-ish key, unique per (post_id, n).
|
|
fh = filehash if filehash is not None else f"{post_id}{n}".encode().hex().ljust(32, "0")[:32]
|
|
return MediaItem(
|
|
url=f"https://cdn.patreon.com/{fh}/{n}.jpg",
|
|
filename=f"{n}.jpg",
|
|
kind=kind,
|
|
filehash=fh,
|
|
post_id=post_id,
|
|
)
|
|
|
|
|
|
class _FakeClient:
|
|
"""Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post
|
|
is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift."""
|
|
|
|
def __init__(self, pages, raise_exc=None):
|
|
self._pages = pages
|
|
self._raise_exc = raise_exc
|
|
self.consumed_posts = 0
|
|
|
|
def iter_posts(self, campaign_id, cursor=None):
|
|
if self._raise_exc is not None:
|
|
raise self._raise_exc
|
|
for page_cursor, posts in self._pages:
|
|
for post_id, media in posts:
|
|
self.consumed_posts += 1
|
|
yield {"id": post_id, "_media": media}, {}, page_cursor
|
|
|
|
def extract_media(self, post, included_index):
|
|
return post["_media"]
|
|
|
|
|
|
class _FakeDownloader:
|
|
"""Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
|
|
`on_disk` report skipped_disk (tier-2); media in `quarantine` report
|
|
quarantined; everything else downloads."""
|
|
|
|
def __init__(self, tmp_path, on_disk=None, quarantine=None, error=None):
|
|
self.tmp_path = tmp_path
|
|
self.on_disk = set(on_disk or ())
|
|
self.quarantine = set(quarantine or ())
|
|
self.error = set(error or ())
|
|
self.download_calls = 0
|
|
|
|
def download_post(self, post, media_items, artist_slug, *, is_seen):
|
|
outcomes = []
|
|
for m in media_items:
|
|
if is_seen(m):
|
|
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
|
elif _ledger_key(m) in self.on_disk:
|
|
p = self.tmp_path / f"{m.post_id}_{m.filename}"
|
|
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
|
|
elif _ledger_key(m) in self.error:
|
|
self.download_calls += 1
|
|
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
|
|
elif _ledger_key(m) in self.quarantine:
|
|
self.download_calls += 1
|
|
q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}"
|
|
outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image"))
|
|
else:
|
|
self.download_calls += 1
|
|
p = self.tmp_path / f"{m.post_id}_{m.filename}"
|
|
p.write_bytes(b"x")
|
|
outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None))
|
|
return outcomes
|
|
|
|
|
|
@pytest.fixture
|
|
async def source_id(db):
|
|
artist = Artist(name="Ingest", slug="ingest")
|
|
db.add(artist)
|
|
await db.flush()
|
|
source = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/ingest", enabled=True, config_overrides={},
|
|
)
|
|
db.add(source)
|
|
await db.commit()
|
|
return source.id
|
|
|
|
|
|
def _ingester(sync_engine, tmp_path, client, downloader):
|
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
|
return PatreonIngester(
|
|
images_root=tmp_path, cookies_path=None, session_factory=factory,
|
|
client=client, downloader=downloader,
|
|
)
|
|
|
|
|
|
def _count_ledger(sync_engine, source_id):
|
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
|
with factory() as s:
|
|
return s.execute(
|
|
select(func.count(PatreonSeenMedia.id)).where(
|
|
PatreonSeenMedia.source_id == source_id
|
|
)
|
|
).scalar_one()
|
|
|
|
|
|
# --- tick -----------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_path):
|
|
m1, m2 = _media("p1", 1), _media("p1", 2)
|
|
client = _FakeClient([(None, [("p1", [m1, m2])])])
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.return_code == 0
|
|
assert result.files_downloaded == 2
|
|
assert len(result.written_paths) == 2
|
|
# plan #704: structured run_stats carry the real counts.
|
|
assert result.run_stats["downloaded_count"] == 2
|
|
assert result.posts_processed == 1
|
|
assert _count_ledger(sync_engine, source_id) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_path):
|
|
"""plan #704 (#4): a media that fails validation is reported as quarantined —
|
|
a real files_quarantined count + paths + run_stats — not a blank 0, and is
|
|
NOT written or marked seen."""
|
|
m1, m2 = _media("p1", 1), _media("p1", 2)
|
|
client = _FakeClient([(None, [("p1", [m1, m2])])])
|
|
downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)})
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
assert result.files_downloaded == 1 # only m1
|
|
assert result.files_quarantined == 1 # m2
|
|
assert len(result.quarantined_paths) == 1
|
|
assert result.run_stats["quarantined_count"] == 1
|
|
assert result.run_stats["downloaded_count"] == 1
|
|
assert len(result.written_paths) == 1 # quarantined NOT written
|
|
# Quarantined media is NOT marked seen (a fixed file may be re-fetched).
|
|
assert _count_ledger(sync_engine, source_id) == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
|
|
m1 = _media("p1", 1)
|
|
# Pre-seed the ledger with m1's key.
|
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
|
with factory() as s:
|
|
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
|
|
s.commit()
|
|
|
|
client = _FakeClient([(None, [("p1", [m1])])])
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
assert result.files_downloaded == 0
|
|
assert downloader.download_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
|
|
# Three posts, one already-seen media each; threshold 2 → stop before the 3rd.
|
|
seen_media = [_media(f"p{i}", 1) for i in range(1, 4)]
|
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
|
with factory() as s:
|
|
for m in seen_media:
|
|
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
|
|
s.commit()
|
|
|
|
pages = [(None, [(m.post_id, [m]) for m in seen_media])]
|
|
client = _FakeClient(pages)
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick", seen_threshold=2,
|
|
)
|
|
assert result.success is True
|
|
# Stopped after the 2nd contiguous seen item — never consumed the 3rd post.
|
|
assert client.consumed_posts == 2
|
|
|
|
|
|
# --- backfill -------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, tmp_path):
|
|
pages = [
|
|
(None, [("p1", [_media("p1", 1)])]),
|
|
("CUR2", [("p2", [_media("p2", 1)])]),
|
|
]
|
|
client = _FakeClient(pages)
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="backfill",
|
|
)
|
|
# rc 0 + error_type None is what _apply_backfill_lifecycle reads as "complete".
|
|
assert result.return_code == 0
|
|
assert result.error_type is None
|
|
assert result.files_downloaded == 2
|
|
# The page-2 cursor is carried structurally for checkpointing (plan #704).
|
|
assert result.cursor == "CUR2"
|
|
assert result.posts_processed == 2
|
|
assert result.run_stats["downloaded_count"] == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_path, db):
|
|
"""plan #705 #6: the cursor is persisted to the DB at each page boundary
|
|
DURING the walk (not just at the end via phase 3), so a worker SIGKILL
|
|
mid-chunk resumes near the crash. After the walk the source carries the last
|
|
page's cursor — written by the ingester, not phase 3 (which the unit run
|
|
never reaches)."""
|
|
pages = [
|
|
(None, [("p1", [_media("p1", 1)])]),
|
|
("CUR2", [("p2", [_media("p2", 1)])]),
|
|
("CUR3", [("p3", [_media("p3", 1)])]),
|
|
]
|
|
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
|
|
ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="backfill",
|
|
)
|
|
co = (await db.execute(
|
|
select(Source.config_overrides).where(Source.id == source_id)
|
|
)).scalar_one()
|
|
assert co.get("_backfill_cursor") == "CUR3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_persists_live_posts_count(source_id, sync_engine, tmp_path, db):
|
|
"""plan #704 #5: the ingester writes a monotonic _backfill_posts absolute
|
|
DURING the walk (seeded from prior chunks via posts_base), EXCLUDING the
|
|
re-walked resume page so the count doesn't inflate across chunks."""
|
|
# Resume from CUR2 (a prior chunk left off here): its page is re-walked and
|
|
# must NOT be re-counted. posts_base=4 stands in for the prior chunks.
|
|
pages = [
|
|
("CUR2", [("p1", [_media("p1", 1)])]), # the resumed page — not counted
|
|
("CUR3", [("p2", [_media("p2", 1)])]), # net-new
|
|
("CUR4", [("p3", [_media("p3", 1)])]), # net-new
|
|
]
|
|
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
|
|
ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="backfill",
|
|
resume_cursor="CUR2", posts_base=4,
|
|
)
|
|
co = (await db.execute(
|
|
select(Source.config_overrides).where(Source.id == source_id)
|
|
)).scalar_one()
|
|
# 4 prior + 2 net-new (p2, p3); p1 on the resumed CUR2 page is excluded.
|
|
assert co.get("_backfill_posts") == 6
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tick_does_not_persist_posts(source_id, sync_engine, tmp_path, db):
|
|
"""A tick has no resumable backfill state, so it must not write _backfill_posts
|
|
(the badge is a backfill/recovery concept)."""
|
|
ing = _ingester(
|
|
sync_engine, tmp_path,
|
|
_FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
|
|
_FakeDownloader(tmp_path),
|
|
)
|
|
ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
co = (await db.execute(
|
|
select(Source.config_overrides).where(Source.id == source_id)
|
|
)).scalar_one()
|
|
assert "_backfill_posts" not in (co or {})
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db):
|
|
"""A tick has no resumable backfill state, so it must not write a cursor."""
|
|
ing = _ingester(
|
|
sync_engine, tmp_path,
|
|
_FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
|
|
_FakeDownloader(tmp_path),
|
|
)
|
|
ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
co = (await db.execute(
|
|
select(Source.config_overrides).where(Source.id == source_id)
|
|
)).scalar_one()
|
|
assert "_backfill_cursor" not in (co or {})
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_budget_cut_returns_partial_with_progress(
|
|
source_id, sync_engine, tmp_path, monkeypatch,
|
|
):
|
|
# Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget).
|
|
# run() lives in ingest_core now, so patch the clock there (the same global
|
|
# time module object, but we reference it through the module that uses it).
|
|
import backend.app.services.ingest_core as core
|
|
ticks = iter([0.0, 10.0, 200.0, 250.0])
|
|
last = [0.0]
|
|
|
|
def fake_monotonic():
|
|
try:
|
|
last[0] = next(ticks)
|
|
except StopIteration:
|
|
pass
|
|
return last[0]
|
|
|
|
monkeypatch.setattr(core.time, "monotonic", fake_monotonic)
|
|
|
|
pages = [
|
|
("CUR1", [("p1", [_media("p1", 1)])]),
|
|
("CUR2", [("p2", [_media("p2", 1)])]),
|
|
]
|
|
client = _FakeClient(pages)
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="backfill",
|
|
resume_cursor=None, time_budget_seconds=100.0,
|
|
)
|
|
assert result.error_type == ErrorType.PARTIAL
|
|
assert result.return_code == -1
|
|
assert result.files_downloaded == 1 # post1 downloaded before the cut
|
|
assert downloader.download_calls == 1 # post2's body never ran
|
|
# The checkpoint is the page we were CUT ON (CUR2 — entered, cursor emitted,
|
|
# then the budget check broke before processing it); the next chunk resumes
|
|
# there. Structured (plan #704), matching the old parse_last_cursor(last).
|
|
assert result.cursor == "CUR2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db):
|
|
"""plan #708 B4: _still_running reflects the source's `_backfill_state` — the
|
|
flag an operator Stop pops to cancel a live chunk."""
|
|
ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path))
|
|
assert ing._still_running(source_id) is False # absent → not running
|
|
src = (await db.execute(select(Source).where(Source.id == source_id))).scalar_one()
|
|
src.config_overrides = {"_backfill_state": "running"}
|
|
await db.commit()
|
|
assert ing._still_running(source_id) is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_stops_when_operator_cancels_mid_walk(
|
|
source_id, sync_engine, tmp_path,
|
|
):
|
|
"""plan #708 B4: with the running state latched, a later page boundary that
|
|
finds it gone (Stop) bails the chunk with PARTIAL instead of finishing."""
|
|
pages = [
|
|
("CUR2", [("p1", [_media("p1", 1)])]),
|
|
("CUR3", [("p2", [_media("p2", 1)])]),
|
|
("CUR4", [("p3", [_media("p3", 1)])]),
|
|
]
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), downloader)
|
|
|
|
# Latch on the page-2 boundary (running), then "Stop" before page 3.
|
|
calls = {"n": 0}
|
|
|
|
def fake_still_running(_source_id):
|
|
calls["n"] += 1
|
|
return calls["n"] == 1
|
|
|
|
ing._still_running = fake_still_running
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="backfill",
|
|
)
|
|
assert result.error_type == ErrorType.PARTIAL
|
|
assert "Stopped by operator" in result.error_message
|
|
assert downloader.download_calls == 1 # only p1 ran; cancelled before p2
|
|
|
|
|
|
# --- recovery -------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path):
|
|
m1 = _media("p1", 1)
|
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
|
with factory() as s:
|
|
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
|
|
s.commit()
|
|
|
|
client = _FakeClient([(None, [("p1", [m1])])])
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="recovery",
|
|
)
|
|
# Ledger says seen, but recovery bypasses tier-1 → re-downloaded for a fresh
|
|
# pHash evaluation under the current threshold.
|
|
assert result.files_downloaded == 1
|
|
assert downloader.download_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path):
|
|
m1 = _media("p1", 1)
|
|
client = _FakeClient([(None, [("p1", [m1])])])
|
|
# File still on disk (a kept image) → tier-2 spares it even under recovery.
|
|
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="recovery",
|
|
)
|
|
assert result.files_downloaded == 0
|
|
assert downloader.download_calls == 0
|
|
# Disk-skip still reconciles the ledger so a later tick skips at tier-1.
|
|
assert _count_ledger(sync_engine, source_id) == 1
|
|
|
|
|
|
# --- dead-letter ledger (plan #705 #7) ------------------------------------
|
|
|
|
|
|
def _count_failed(sync_engine, source_id):
|
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
|
with factory() as s:
|
|
return s.execute(
|
|
select(func.count(PatreonFailedMedia.id)).where(
|
|
PatreonFailedMedia.source_id == source_id
|
|
)
|
|
).scalar_one()
|
|
|
|
|
|
def _seed_failed(sync_engine, source_id, key, attempts):
|
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
|
with factory() as s:
|
|
s.add(PatreonFailedMedia(source_id=source_id, filehash=key, attempts=attempts))
|
|
s.commit()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, tmp_path):
|
|
"""A media that errors DEAD_LETTER_THRESHOLD times is recorded; the next walk
|
|
skips it (no download attempt) and counts it dead-lettered."""
|
|
from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
|
|
|
|
m1 = _media("p1", 1)
|
|
|
|
def _run():
|
|
ing = _ingester(
|
|
sync_engine, tmp_path,
|
|
_FakeClient([(None, [("p1", [m1])])]),
|
|
_FakeDownloader(tmp_path, error={_ledger_key(m1)}),
|
|
)
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
return result, ing.downloader
|
|
|
|
# Fail it up to the threshold — each run attempts the download and errors.
|
|
for _ in range(DEAD_LETTER_THRESHOLD):
|
|
result, dl = _run()
|
|
assert dl.download_calls == 1
|
|
assert _count_failed(sync_engine, source_id) == 1
|
|
|
|
# Now attempts == threshold → dead: the next walk skips it without trying.
|
|
result, dl = _run()
|
|
assert dl.download_calls == 0
|
|
assert result.run_stats["dead_lettered_count"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recovery_reattempts_dead_lettered_and_clears_on_success(
|
|
source_id, sync_engine, tmp_path,
|
|
):
|
|
"""Recovery bypasses the dead-letter ledger (re-attempts dead media); a clean
|
|
download clears the row (it recovered)."""
|
|
from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
|
|
|
|
m1 = _media("p1", 1)
|
|
_seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD)
|
|
|
|
downloader = _FakeDownloader(tmp_path) # succeeds
|
|
ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="recovery",
|
|
)
|
|
assert downloader.download_calls == 1 # re-attempted despite being dead
|
|
assert result.files_downloaded == 1
|
|
assert _count_failed(sync_engine, source_id) == 0 # cleared on success
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clean_download_clears_below_threshold_failure(
|
|
source_id, sync_engine, tmp_path,
|
|
):
|
|
"""A media with a sub-threshold failure that now downloads cleanly (tick) has
|
|
its dead-letter row cleared."""
|
|
m1 = _media("p1", 1)
|
|
_seed_failed(sync_engine, source_id, _ledger_key(m1), 1) # not yet dead
|
|
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
|
|
result = ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
assert result.files_downloaded == 1
|
|
assert _count_failed(sync_engine, source_id) == 0
|
|
|
|
|
|
# --- ledger + drift -------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
|
|
m1 = _media("p1", 1)
|
|
client = _FakeClient([(None, [("p1", [m1])])])
|
|
downloader = _FakeDownloader(tmp_path)
|
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
|
|
|
# Same media downloaded twice across two runs — the second marks seen again.
|
|
ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
|
|
ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
|
|
assert _count_ledger(sync_engine, source_id) == 1
|
|
|
|
|
|
def _run_with_client_error(source_id, sync_engine, tmp_path, exc):
|
|
client = _FakeClient([], raise_exc=exc)
|
|
ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
|
|
return ing.run(
|
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
|
url="https://patreon.com/ingest", mode="tick",
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_drift_maps_to_api_drift_and_fails_loud(source_id, sync_engine, tmp_path):
|
|
result = _run_with_client_error(
|
|
source_id, sync_engine, tmp_path,
|
|
PatreonDriftError("missing top-level 'data' key"),
|
|
)
|
|
assert result.success is False
|
|
assert result.error_type == ErrorType.API_DRIFT
|
|
assert "Patreon API changed" in (result.error_message or "")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path):
|
|
result = _run_with_client_error(
|
|
source_id, sync_engine, tmp_path,
|
|
PatreonAuthError("HTTP 403 — auth rejected", status_code=403),
|
|
)
|
|
assert result.success is False
|
|
assert result.error_type == ErrorType.AUTH_ERROR
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rate_limit_status_maps_to_rate_limited(source_id, sync_engine, tmp_path):
|
|
result = _run_with_client_error(
|
|
source_id, sync_engine, tmp_path,
|
|
PatreonAPIError("HTTP 429", status_code=429),
|
|
)
|
|
assert result.error_type == ErrorType.RATE_LIMITED
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_not_found_status_maps_to_not_found(source_id, sync_engine, tmp_path):
|
|
result = _run_with_client_error(
|
|
source_id, sync_engine, tmp_path,
|
|
PatreonAPIError("HTTP 404", status_code=404),
|
|
)
|
|
assert result.error_type == ErrorType.NOT_FOUND
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_transport_error_maps_to_network_error(source_id, sync_engine, tmp_path):
|
|
result = _run_with_client_error(
|
|
source_id, sync_engine, tmp_path,
|
|
PatreonAPIError("connection reset"), # no status_code → transport
|
|
)
|
|
assert result.error_type == ErrorType.NETWORK_ERROR
|
|
|
|
|
|
def test_ledger_key_video_has_no_filehash():
|
|
video = MediaItem(
|
|
url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
|
|
kind="postfile", filehash=None, post_id="p9",
|
|
)
|
|
assert _ledger_key(video) == "p9:clip.mp4"
|
|
|
|
|
|
# --- verify_patreon_credential (the verify counterpart, plan #697) ---------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_verify_patreon_credential_unresolvable_is_inconclusive():
|
|
from backend.app.services.patreon_ingester import verify_patreon_credential
|
|
|
|
# No campaign id resolvable from a non-Patreon URL → can't test → None.
|
|
ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {})
|
|
assert ok is None
|
|
assert "campaign id" in msg.lower()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
|
|
import backend.app.services.patreon_ingester as mod
|
|
|
|
async def _fake_resolve(url, cookies_path, overrides):
|
|
return "123", None
|
|
monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve)
|
|
monkeypatch.setattr(
|
|
mod.PatreonClient, "verify_auth",
|
|
lambda self, campaign_id: (True, f"ok:{campaign_id}"),
|
|
)
|
|
ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
|
|
assert ok is True
|
|
assert msg == "ok:123"
|