bvandeusen
5b615b7ded
Before arming a deep walk on a native-ingester platform (Patreon — where
verify is one cheap API page), POST /sources/{id}/backfill {start|recover}
runs the shared verify_source_credential first and REFUSES (409 + reason)
only on a definitive rejection (verify→False, e.g. expired cookies). It
proceeds on valid (True) or inconclusive (None — a network blip must not
block). Gated to native platforms: gallery-dl verify is a slow --simulate
subprocess, too heavy for an arm action. The credential read happens in a
session that's CLOSED before the verify network call (no held conn).
Frontend: onBackfill/onRecover now read e.body.detail (ApiError carries the
reason in .body, not .detail) so the rejection text surfaces in the toast.
Tests: arm blocked on rejection (409, source not armed), proceeds on
inconclusive, stop never pre-flights, gallery-dl platform skips pre-flight.
An autouse fixture stubs verify to 'valid' for the existing backfill
endpoint tests so they stay network-free.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
381 lines
13 KiB
Python
381 lines
13 KiB
Python
import pytest
|
|
|
|
from backend.app.models import Artist, Source
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _stub_preflight_verify(monkeypatch):
|
|
"""Backfill/recover arms run a pre-flight credential verify (plan #703 #2)
|
|
which for Patreon would hit the network. Default it to 'valid' so the
|
|
endpoint tests stay network-free; the dedicated pre-flight tests override
|
|
this with a rejection/inconclusive verdict."""
|
|
from backend.app.services import download_backends as db_mod
|
|
|
|
async def _ok(**kwargs):
|
|
return (True, "Credentials valid")
|
|
|
|
monkeypatch.setattr(db_mod, "verify_source_credential", _ok)
|
|
|
|
|
|
@pytest.fixture
|
|
async def artist(db):
|
|
a = Artist(name="Alice", slug="alice")
|
|
db.add(a)
|
|
await db.commit()
|
|
return a
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_failing_filter_returns_only_failing(client, artist, db):
|
|
healthy = Source(
|
|
artist_id=artist.id, platform="patreon", url="https://p/ok",
|
|
enabled=True, consecutive_failures=0,
|
|
)
|
|
broken = Source(
|
|
artist_id=artist.id, platform="pixiv", url="https://p/bad",
|
|
enabled=True, consecutive_failures=3, last_error="boom",
|
|
)
|
|
db.add_all([healthy, broken])
|
|
await db.commit()
|
|
|
|
resp = await client.get("/api/sources?failing=true")
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert len(body) == 1
|
|
assert body[0]["consecutive_failures"] == 3
|
|
assert body[0]["last_error"] == "boom"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_schedule_status_shape(client):
|
|
resp = await client.get("/api/sources/schedule-status")
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert set(body) == {
|
|
"last_tick_at", "next_due_at", "due_now", "auto_sources",
|
|
"platform_cooldowns",
|
|
}
|
|
assert isinstance(body["due_now"], int)
|
|
assert isinstance(body["auto_sources"], int)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_list_get_delete(client, artist):
|
|
create = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "patreon",
|
|
"url": "https://patreon.com/alice",
|
|
})
|
|
assert create.status_code == 201
|
|
created = await create.get_json()
|
|
assert created["artist_name"] == "Alice"
|
|
sid = created["id"]
|
|
|
|
listing = await client.get("/api/sources")
|
|
assert listing.status_code == 200
|
|
assert len(await listing.get_json()) == 1
|
|
|
|
filtered = await client.get(f"/api/sources?artist_id={artist.id}")
|
|
assert filtered.status_code == 200
|
|
assert len(await filtered.get_json()) == 1
|
|
|
|
one = await client.get(f"/api/sources/{sid}")
|
|
assert one.status_code == 200
|
|
assert (await one.get_json())["id"] == sid
|
|
|
|
deleted = await client.delete(f"/api/sources/{sid}")
|
|
assert deleted.status_code == 204
|
|
assert (await client.get(f"/api/sources/{sid}")).status_code == 404
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_rejects_unknown_platform(client, artist):
|
|
resp = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "myspace", "url": "https://m/x",
|
|
})
|
|
assert resp.status_code == 400
|
|
body = await resp.get_json()
|
|
assert body["error"] == "unknown_platform"
|
|
assert "known" in body
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_rejects_bad_config(client, artist):
|
|
resp = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "patreon",
|
|
"url": "https://p/a", "config_overrides": [1, 2, 3],
|
|
})
|
|
assert resp.status_code == 400
|
|
assert (await resp.get_json())["error"] == "invalid_config"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_rejects_empty_url(client, artist):
|
|
resp = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "patreon", "url": " ",
|
|
})
|
|
assert resp.status_code == 400
|
|
assert (await resp.get_json())["error"] == "empty_url"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_rejects_unknown_artist(client):
|
|
resp = await client.post("/api/sources", json={
|
|
"artist_id": 99999, "platform": "patreon", "url": "https://p/a",
|
|
})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_duplicate_returns_409_with_existing_id(client, artist):
|
|
a = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "patreon", "url": "https://p/a",
|
|
})
|
|
first = (await a.get_json())["id"]
|
|
b = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "patreon", "url": "https://p/a",
|
|
})
|
|
assert b.status_code == 409
|
|
body = await b.get_json()
|
|
assert body["error"] == "duplicate"
|
|
assert body["existing_id"] == first
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_updates_fields(client, artist):
|
|
create = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "patreon", "url": "https://p/a",
|
|
})
|
|
sid = (await create.get_json())["id"]
|
|
patch = await client.patch(f"/api/sources/{sid}", json={"enabled": False})
|
|
assert patch.status_code == 200
|
|
assert (await patch.get_json())["enabled"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_404(client):
|
|
assert (await client.get("/api/sources/99999")).status_code == 404
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_404(client):
|
|
assert (await client.patch("/api/sources/99999", json={"enabled": False})).status_code == 404
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_404(client):
|
|
assert (await client.delete("/api/sources/99999")).status_code == 404
|
|
|
|
|
|
# --- FC-3d: SourceRecord surfaces consecutive_failures + next_check_at ----
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_response_includes_fc3d_fields(client, artist):
|
|
resp = await client.post("/api/sources", json={
|
|
"artist_id": artist.id, "platform": "patreon",
|
|
"url": "https://patreon.com/alice-fc3d-new",
|
|
})
|
|
assert resp.status_code == 201
|
|
body = await resp.get_json()
|
|
assert body["consecutive_failures"] == 0
|
|
assert body["next_check_at"] is None # never checked yet
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_derives_next_check_at_when_last_checked_set(
|
|
client, artist, db,
|
|
):
|
|
from datetime import UTC, datetime
|
|
|
|
from backend.app.models import Source
|
|
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-fc3d-checked", enabled=True,
|
|
consecutive_failures=0,
|
|
last_checked_at=datetime.now(UTC),
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
|
|
resp = await client.get("/api/sources")
|
|
body = await resp.get_json()
|
|
target = next(
|
|
r for r in body if r["url"] == "https://patreon.com/alice-fc3d-checked"
|
|
)
|
|
assert target["next_check_at"] is not None
|
|
assert "T" in target["next_check_at"] # ISO 8601
|
|
|
|
|
|
# --- Plan #544: POST /api/sources/{id}/backfill ----------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_endpoint_start_and_stop(client, artist, db):
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-backfill", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
sid = src.id
|
|
|
|
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "start"})
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert body["backfill_state"] == "running"
|
|
|
|
# GET reflects the running state.
|
|
one = await client.get(f"/api/sources/{sid}")
|
|
assert (await one.get_json())["backfill_state"] == "running"
|
|
|
|
stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
|
|
assert (await stopped.get_json())["backfill_state"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_endpoint_defaults_to_start(client, artist, db):
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-backfill-default", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
resp = await client.post(f"/api/sources/{src.id}/backfill", json={})
|
|
body = await resp.get_json()
|
|
assert body["backfill_state"] == "running"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_endpoint_rejects_bad_action(client, artist, db):
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-backfill-bad", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
bad = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "nope"})
|
|
assert bad.status_code == 400
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_endpoint_404_when_source_missing(client):
|
|
resp = await client.post("/api/sources/999999/backfill", json={"action": "start"})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_endpoint_recover_arms_bypass(client, artist, db):
|
|
"""Plan #697: action='recover' arms a backfill that bypasses the Patreon
|
|
seen-ledger; the response exposes backfill_bypass_seen=True so the UI badge
|
|
can label it 'Recovering'. Stop clears it."""
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-recover", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
sid = src.id
|
|
|
|
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"})
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert body["backfill_state"] == "running"
|
|
assert body["backfill_bypass_seen"] is True
|
|
|
|
stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
|
|
stopped_body = await stopped.get_json()
|
|
assert stopped_body["backfill_state"] is None
|
|
assert stopped_body["backfill_bypass_seen"] is False
|
|
|
|
|
|
# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm -----
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_arm_blocked_when_credential_rejected(client, artist, db, monkeypatch):
|
|
"""A definitively-rejected credential refuses the arm (409 + reason) instead
|
|
of starting a doomed walk; the source is NOT armed."""
|
|
from backend.app.services import download_backends as db_mod
|
|
|
|
async def _reject(**kwargs):
|
|
return (False, "Patreon rejected the credential — cookies expired")
|
|
monkeypatch.setattr(db_mod, "verify_source_credential", _reject)
|
|
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-preflight", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
sid = src.id
|
|
|
|
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"})
|
|
assert resp.status_code == 409
|
|
assert "expired" in ((await resp.get_json()).get("detail") or "")
|
|
|
|
# Not armed.
|
|
one = await (await client.get(f"/api/sources/{sid}")).get_json()
|
|
assert one["backfill_state"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_arm_proceeds_when_credential_inconclusive(client, artist, db, monkeypatch):
|
|
"""An inconclusive verify (network blip / drift) must NOT block the arm."""
|
|
from backend.app.services import download_backends as db_mod
|
|
|
|
async def _inconclusive(**kwargs):
|
|
return (None, "couldn't verify (network)")
|
|
monkeypatch.setattr(db_mod, "verify_source_credential", _inconclusive)
|
|
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-incon", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
|
|
resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"})
|
|
assert resp.status_code == 200
|
|
assert (await resp.get_json())["backfill_state"] == "running"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stop_never_pre_flights(client, artist, db, monkeypatch):
|
|
from backend.app.services import download_backends as db_mod
|
|
|
|
async def _boom(**kwargs):
|
|
raise AssertionError("stop must not pre-flight verify")
|
|
monkeypatch.setattr(db_mod, "verify_source_credential", _boom)
|
|
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/alice-stop", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "stop"})
|
|
assert resp.status_code == 200
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_gallery_dl_platform_arm_skips_pre_flight(client, artist, db, monkeypatch):
|
|
"""Pre-flight is gated to native-ingester platforms (cheap verify); a
|
|
gallery-dl source arms without the slow --simulate probe."""
|
|
from backend.app.services import download_backends as db_mod
|
|
|
|
async def _boom(**kwargs):
|
|
raise AssertionError("gallery-dl arm must not pre-flight verify")
|
|
monkeypatch.setattr(db_mod, "verify_source_credential", _boom)
|
|
|
|
src = Source(
|
|
artist_id=artist.id, platform="subscribestar",
|
|
url="https://www.subscribestar.com/x", enabled=True,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"})
|
|
assert resp.status_code == 200
|