diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 0e817c4..6d104e6 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -354,18 +354,35 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict: Returns: dry_run=True: {"count": N, "sample_names": [first 50]} dry_run=False: {"deleted": N, "sample_names": [first 50]} + + Implementation note: the previous SELECT-ids → DELETE-WHERE-IN + pattern was vulnerable to the psycopg 65535-parameter ceiling on + libraries with tag explosions. The live delete now runs a single + DELETE with the same NOT-IN predicate find_unused_tags uses, so + the row count scales without binding every id as a parameter. + Audit 2026-06-02. """ - unused = find_unused_tags(session) - sample = [t.name for t in unused[:50]] + sample_rows = find_unused_tags(session, limit=50) + sample = [t.name for t in sample_rows] + used_via_image_tag = select(image_tag.c.tag_id).distinct() + used_via_series = select(SeriesPage.series_tag_id).where( + SeriesPage.series_tag_id.is_not(None) + ).distinct() if dry_run: - return {"count": len(unused), "sample_names": sample} - ids = [t.id for t in unused] - if ids: - session.execute( - Tag.__table__.delete().where(Tag.id.in_(ids)) - ) - session.commit() - return {"deleted": len(ids), "sample_names": sample} + count = session.execute( + select(func.count()) + .select_from(Tag) + .where(Tag.id.not_in(used_via_image_tag)) + .where(Tag.id.not_in(used_via_series)) + ).scalar_one() + return {"count": count, "sample_names": sample} + result = session.execute( + Tag.__table__.delete() + .where(Tag.id.not_in(used_via_image_tag)) + .where(Tag.id.not_in(used_via_series)) + ) + session.commit() + return {"deleted": result.rowcount or 0, "sample_names": sample} # Legacy tags FC no longer uses, in two shapes: diff --git a/backend/app/services/extension_service.py b/backend/app/services/extension_service.py index 4c138de..e6fa7a6 100644 --- a/backend/app/services/extension_service.py +++ b/backend/app/services/extension_service.py @@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from ..models import Artist, Source from ..utils.slug import slugify +from .source_service import NEW_SOURCE_BACKFILL_RUNS class UnknownPlatformError(Exception): @@ -204,9 +205,16 @@ class ExtensionService: return existing, False sp = await self.session.begin_nested() try: + # New subscription sources arm a few backfill runs so the + # first ticks walk the full history (otherwise gallery-dl's + # exit:20 short-circuits before the archive is built). + # Mirrors SourceService.create — without it, Firefox quick- + # add on a creator with >20 unsynced posts would surface + # as "check failed" with no diagnosis. Audit 2026-06-02. src = Source( artist_id=artist_id, platform=platform, url=url, enabled=True, + backfill_runs_remaining=NEW_SOURCE_BACKFILL_RUNS, ) self.session.add(src) await self.session.flush() diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 0f19793..8e6a434 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -58,8 +58,19 @@ TICK_SKIP_VALUE = "exit:20" # Backfill mode (operator-triggered deep scan): walk the full history. # Source.backfill_runs_remaining > 0 selects this mode; the longer # timeout below absorbs creators with thousands of posts. +# +# 30 seconds shy of Celery's hard `time_limit=1200` on download_source +# (tasks/download.py:33). subprocess.run MUST raise TimeoutExpired +# before Celery SIGKILLs the worker — same rationale as the tick +# default at line 74. The audit (2026-06-02) caught this at 1800, +# guaranteeing SIGKILL on any backfill that ran to its subprocess +# budget: stdout/stderr lost, backfill_runs_remaining never +# decrements, recovery sweep stamps generic "stranded" 30 min later. +# Recreates the exact Knuxy #38275 failure mode the tick 870s default +# was added to prevent. backfill_runs_remaining=3 still gives ~58 +# minutes of cumulative walk across three runs for prolific creators. BACKFILL_SKIP_VALUE = True -BACKFILL_TIMEOUT_SECONDS = 1800 +BACKFILL_TIMEOUT_SECONDS = 1170 # 30 seconds shy of download_source's Celery soft_time_limit (900s, see @@ -844,7 +855,13 @@ class GalleryDLService: ), ) etype, msg = self._categorize_error(proc.returncode, proc.stdout, proc.stderr) - if proc.returncode == 0 or etype == ErrorType.NO_NEW_CONTENT: + # TIER_LIMITED proves auth worked — gallery-dl reached the + # post, was told it's tier-gated. The download path treats + # this as success (line 712); verify must too, or operators + # rotate working cookies for no reason. Audit 2026-06-02. + if proc.returncode == 0 or etype in ( + ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED, + ): return True, "Credentials valid — the feed authenticated." if etype == ErrorType.AUTH_ERROR: return False, msg diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 144a994..1e2031c 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -90,7 +90,7 @@ _EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "p # Plan #544 follow-up: newly created enabled sources pre-arm backfill so # their first N polls walk gallery-dl's full post history with the longer # timeout (matches the manual "Deep scan" button's default). -_NEW_SOURCE_BACKFILL_RUNS = 3 +NEW_SOURCE_BACKFILL_RUNS = 3 class SourceService: @@ -218,7 +218,7 @@ class SourceService: # the budget is spent or the queue drains. # Disabled sources (incl. sidecar synthetics, url='sidecar:...') # are never polled, so leave their counter at 0. - backfill_runs = _NEW_SOURCE_BACKFILL_RUNS if enabled else 0 + backfill_runs = NEW_SOURCE_BACKFILL_RUNS if enabled else 0 source = Source( artist_id=artist_id, platform=platform, url=url, enabled=enabled, config_overrides=config_overrides, diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 1c4cf22..567e97b 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -7,7 +7,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from PIL import Image -from sqlalchemy import and_, delete, or_, select, update +from sqlalchemy import Integer, and_, cast, delete, func, or_, select, update from ..celery_app import celery from ..models import ( @@ -182,6 +182,11 @@ def recover_interrupted_tasks() -> int: .where(ImportTask.created_at < orphan_cutoff) .values( status="failed", + # Without finished_at, cleanup_old_tasks (`WHERE + # finished_at < cutoff`) never reaps these rows — + # orphan-swept rows would become permanent table + # tenants. Audit 2026-06-02. + finished_at=now, error=( "orphan pending/queued swept by recover_interrupted_tasks " "(scanner likely crashed mid-enqueue); retry via " @@ -278,6 +283,14 @@ def recover_stalled_task_runs() -> int: f"no completion signal received within {minutes} min" ), finished_at=now, + # Matches celery_signals.finalize's + # int((now - started_at).total_seconds() * 1000) + # — sweep-closed rows now carry duration like + # normally-finalized rows. Audit 2026-06-02. + duration_ms=cast( + func.extract("epoch", now - TaskRun.started_at) * 1000, + Integer, + ), ) ) for w in extra_where: diff --git a/frontend/src/stores/credentials.js b/frontend/src/stores/credentials.js index c3558e3..c9160a2 100644 --- a/frontend/src/stores/credentials.js +++ b/frontend/src/stores/credentials.js @@ -23,7 +23,11 @@ export const useCredentialsStore = defineStore('credentials', () => { const rec = await api.post('/api/credentials', { body: { platform, credential_type, data }, }) - byPlatform.value.delete(platform) + // Reflect the returned record immediately — the previous .delete() + // call left the card rendering "no credential" for the gap between + // upload completion and the caller's follow-up loadAll(). Audit + // 2026-06-02. + byPlatform.value.set(platform, rec) return rec } diff --git a/frontend/test/credentials.spec.js b/frontend/test/credentials.spec.js index 8437726..5b8d8fb 100644 --- a/frontend/test/credentials.spec.js +++ b/frontend/test/credentials.spec.js @@ -27,12 +27,17 @@ describe('credentials store', () => { expect(s.byPlatform.get('patreon').credential_type).toBe('cookies') }) - it('upload invalidates the cache', async () => { + it('upload reflects the returned record into the cache', async () => { + // The previous behavior was to delete the cache entry on upload, which + // briefly showed "no credential" until a follow-up loadAll() resolved. + // Audit 2026-06-02 corrected this: upload now stores the returned + // record directly so the card updates immediately. const s = useCredentialsStore() - s.byPlatform.set('patreon', { platform: 'patreon' }) - stubFetch(() => ({ status: 201, body: { platform: 'patreon', credential_type: 'cookies' } })) + s.byPlatform.set('patreon', { platform: 'patreon', credential_type: 'token' }) + const fresh = { platform: 'patreon', credential_type: 'cookies' } + stubFetch(() => ({ status: 201, body: fresh })) await s.upload('patreon', 'cookies', 'NETSCAPE_CONTENT') - expect(s.byPlatform.has('patreon')).toBe(false) + expect(s.byPlatform.get('patreon')).toEqual(fresh) }) it('remove deletes and invalidates the cache', async () => {