Post.source_id refactor + tick/backfill modes + PARTIAL classifier + Mux fix + UX #44

Merged
bvandeusen merged 9 commits from dev into main 2026-06-01 20:44:24 -04:00
2 changed files with 44 additions and 0 deletions
Showing only changes of commit 2aa2002f22 - Show all commits
+16
View File
@@ -87,6 +87,11 @@ class SourceRecord:
_EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"}
# Plan #544 follow-up: newly created enabled sources pre-arm backfill so
# their first N polls walk gallery-dl's full post history with the longer
# timeout (matches the manual "Deep scan" button's default).
_NEW_SOURCE_BACKFILL_RUNS = 3
class SourceService:
def __init__(self, session: AsyncSession):
@@ -204,10 +209,21 @@ class SourceService:
select(func.count(Source.id)).where(Source.artist_id == artist_id)
)).scalar_one()
# Plan #544 follow-up: a freshly added subscription has no archive
# yet, so the first few polls would walk the full post history in
# tick mode and trip exit:20 after ~20 contiguous archive hits —
# except there are none yet, so tick mode would walk forever and
# blow the wall-clock cap. Pre-arm backfill so the initial syncs
# use the longer timeout + skip:True walk. Tick mode resumes once
# the budget is spent or the queue drains.
# Disabled sources (incl. sidecar synthetics, url='sidecar:...')
# are never polled, so leave their counter at 0.
backfill_runs = _NEW_SOURCE_BACKFILL_RUNS if enabled else 0
source = Source(
artist_id=artist_id, platform=platform, url=url,
enabled=enabled, config_overrides=config_overrides,
check_interval_override=check_interval_override,
backfill_runs_remaining=backfill_runs,
)
self.session.add(source)
try:
+28
View File
@@ -213,3 +213,31 @@ async def test_set_backfill_runs_raises_when_source_missing(db):
svc = SourceService(db)
with pytest.raises(LookupError):
await svc.set_backfill_runs(99999, 3)
@pytest.mark.asyncio
async def test_new_enabled_source_starts_in_backfill_mode(db):
"""Plan #544 follow-up: freshly added enabled sources have no archive
yet, so the first few polls would blow the wall-clock cap in tick
mode. Pre-arm backfill so the initial walks use the longer timeout."""
artist = await _artist(db, "Alice")
svc = SourceService(db)
rec = await svc.create(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice-new",
)
assert rec.backfill_runs_remaining == 3
@pytest.mark.asyncio
async def test_new_disabled_source_skips_backfill(db):
"""Disabled sources (incl. sidecar synthetics that arrive disabled) are
never polled, so don't burn a backfill budget on them."""
artist = await _artist(db, "Alice")
svc = SourceService(db)
rec = await svc.create(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice-disabled",
enabled=False,
)
assert rec.backfill_runs_remaining == 0