bvandeusen/FabledCurator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity
Files
7395e77d75f5fbe712868edcd270df65bc079d9b
FabledCurator/tests/test_api_sources.py
T
bvandeusen 96fffaff64
CI / lint (push) Successful in 2s
Details
CI / backend-lint-and-test (push) Successful in 11s
Details
CI / frontend-build (push) Successful in 17s
Details
CI / integration (push) Successful in 2m58s
Details
feat(download): smarter backfill — time-boxed chunks, run-until-done (backend) 
Plan #693. Large-catalog backfill (Anduo) no longer sprints to the timeout
wall and dies as an error each run. Builds on the cursor checkpoint (#689).

- Time-boxed chunks: BACKFILL_TIMEOUT_SECONDS(1170)→BACKFILL_CHUNK_SECONDS(600),
  far under the 1350 soft limit. Hitting it = normal chunk boundary (the
  TimeoutExpired path already captures partial output + the cursor), not a
  near-wall death.
- Run-until-done state machine driven by config_overrides[_backfill_state]
  (running/complete/stalled). A running backfill auto-continues in chunks
  across ticks until gallery-dl exits cleanly (rc=0 = reached the bottom →
  'complete'); a safety-cap (BACKFILL_MAX_CHUNKS=200) + the #689 stall-guard
  pause a pathological walk as 'stalled'. Replaces the N-runs counter
  (backfill_runs_remaining repurposed as the cap countdown).
- Progress, not error: a chunk that timed out but advanced (cursor moved
  and/or files written) is reclassified TIMEOUT→PARTIAL (status 'ok').
- Retry storm tamed: gallery-dl retries 3→2, downloader timeout 120→60s, so
  one stuck CDN file fails in ~1-2 min not ~10 (Anduo #40838).
- API: POST /sources/{id}/backfill now takes {action: start|stop}; service
  start_backfill/stop_backfill; new enabled sources auto-arm run-until-done;
  source dict exposes backfill_state + backfill_chunks.

Frontend (Start/Stop control + state badge) lands in the next push.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-05 15:02:46 -04:00

253 lines
7.9 KiB
Python
Raw Blame History

import pytest
from backend.app.models import Artist, Source
pytestmark = pytest.mark.integration
@pytest.fixture
async def artist(db):
a = Artist(name="Alice", slug="alice")
db.add(a)
await db.commit()
return a
@pytest.mark.asyncio
async def test_list_failing_filter_returns_only_failing(client, artist, db):
healthy = Source(
artist_id=artist.id, platform="patreon", url="https://p/ok",
enabled=True, consecutive_failures=0,
)
broken = Source(
artist_id=artist.id, platform="pixiv", url="https://p/bad",
enabled=True, consecutive_failures=3, last_error="boom",
)
db.add_all([healthy, broken])
await db.commit()
resp = await client.get("/api/sources?failing=true")
assert resp.status_code == 200
body = await resp.get_json()
assert len(body) == 1
assert body[0]["consecutive_failures"] == 3
assert body[0]["last_error"] == "boom"
@pytest.mark.asyncio
async def test_schedule_status_shape(client):
resp = await client.get("/api/sources/schedule-status")
assert resp.status_code == 200
body = await resp.get_json()
assert set(body) == {
"last_tick_at", "next_due_at", "due_now", "auto_sources",
"platform_cooldowns",
}
assert isinstance(body["due_now"], int)
assert isinstance(body["auto_sources"], int)
@pytest.mark.asyncio
async def test_create_list_get_delete(client, artist):
create = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "patreon",
"url": "https://patreon.com/alice",
})
assert create.status_code == 201
created = await create.get_json()
assert created["artist_name"] == "Alice"
sid = created["id"]
listing = await client.get("/api/sources")
assert listing.status_code == 200
assert len(await listing.get_json()) == 1
filtered = await client.get(f"/api/sources?artist_id={artist.id}")
assert filtered.status_code == 200
assert len(await filtered.get_json()) == 1
one = await client.get(f"/api/sources/{sid}")
assert one.status_code == 200
assert (await one.get_json())["id"] == sid
deleted = await client.delete(f"/api/sources/{sid}")
assert deleted.status_code == 204
assert (await client.get(f"/api/sources/{sid}")).status_code == 404
@pytest.mark.asyncio
async def test_create_rejects_unknown_platform(client, artist):
resp = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "myspace", "url": "https://m/x",
})
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "unknown_platform"
assert "known" in body
@pytest.mark.asyncio
async def test_create_rejects_bad_config(client, artist):
resp = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "patreon",
"url": "https://p/a", "config_overrides": [1, 2, 3],
})
assert resp.status_code == 400
assert (await resp.get_json())["error"] == "invalid_config"
@pytest.mark.asyncio
async def test_create_rejects_empty_url(client, artist):
resp = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "patreon", "url": " ",
})
assert resp.status_code == 400
assert (await resp.get_json())["error"] == "empty_url"
@pytest.mark.asyncio
async def test_create_rejects_unknown_artist(client):
resp = await client.post("/api/sources", json={
"artist_id": 99999, "platform": "patreon", "url": "https://p/a",
})
assert resp.status_code == 404
@pytest.mark.asyncio
async def test_create_duplicate_returns_409_with_existing_id(client, artist):
a = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "patreon", "url": "https://p/a",
})
first = (await a.get_json())["id"]
b = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "patreon", "url": "https://p/a",
})
assert b.status_code == 409
body = await b.get_json()
assert body["error"] == "duplicate"
assert body["existing_id"] == first
@pytest.mark.asyncio
async def test_patch_updates_fields(client, artist):
create = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "patreon", "url": "https://p/a",
})
sid = (await create.get_json())["id"]
patch = await client.patch(f"/api/sources/{sid}", json={"enabled": False})
assert patch.status_code == 200
assert (await patch.get_json())["enabled"] is False
@pytest.mark.asyncio
async def test_get_404(client):
assert (await client.get("/api/sources/99999")).status_code == 404
@pytest.mark.asyncio
async def test_patch_404(client):
assert (await client.patch("/api/sources/99999", json={"enabled": False})).status_code == 404
@pytest.mark.asyncio
async def test_delete_404(client):
assert (await client.delete("/api/sources/99999")).status_code == 404
# --- FC-3d: SourceRecord surfaces consecutive_failures + next_check_at ----
@pytest.mark.asyncio
async def test_create_response_includes_fc3d_fields(client, artist):
resp = await client.post("/api/sources", json={
"artist_id": artist.id, "platform": "patreon",
"url": "https://patreon.com/alice-fc3d-new",
})
assert resp.status_code == 201
body = await resp.get_json()
assert body["consecutive_failures"] == 0
assert body["next_check_at"] is None # never checked yet
@pytest.mark.asyncio
async def test_list_derives_next_check_at_when_last_checked_set(
client, artist, db,
):
from datetime import UTC, datetime
from backend.app.models import Source
src = Source(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice-fc3d-checked", enabled=True,
consecutive_failures=0,
last_checked_at=datetime.now(UTC),
)
db.add(src)
await db.commit()
resp = await client.get("/api/sources")
body = await resp.get_json()
target = next(
r for r in body if r["url"] == "https://patreon.com/alice-fc3d-checked"
)
assert target["next_check_at"] is not None
assert "T" in target["next_check_at"] # ISO 8601
# --- Plan #544: POST /api/sources/{id}/backfill ----------------------------
@pytest.mark.asyncio
async def test_backfill_endpoint_start_and_stop(client, artist, db):
src = Source(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice-backfill", enabled=True,
)
db.add(src)
await db.commit()
sid = src.id
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "start"})
assert resp.status_code == 200
body = await resp.get_json()
assert body["backfill_state"] == "running"
# GET reflects the running state.
one = await client.get(f"/api/sources/{sid}")
assert (await one.get_json())["backfill_state"] == "running"
stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
assert (await stopped.get_json())["backfill_state"] is None
@pytest.mark.asyncio
async def test_backfill_endpoint_defaults_to_start(client, artist, db):
src = Source(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice-backfill-default", enabled=True,
)
db.add(src)
await db.commit()
resp = await client.post(f"/api/sources/{src.id}/backfill", json={})
body = await resp.get_json()
assert body["backfill_state"] == "running"
@pytest.mark.asyncio
async def test_backfill_endpoint_rejects_bad_action(client, artist, db):
src = Source(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice-backfill-bad", enabled=True,
)
db.add(src)
await db.commit()
bad = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "nope"})
assert bad.status_code == 400
@pytest.mark.asyncio
async def test_backfill_endpoint_404_when_source_missing(client):
resp = await client.post("/api/sources/999999/backfill", json={"action": "start"})
assert resp.status_code == 404
Reference in New Issue View Git Blame Copy Permalink