diff --git a/backend/app/services/discord_downloader.py b/backend/app/services/discord_downloader.py index ffd8734..d83e354 100644 --- a/backend/app/services/discord_downloader.py +++ b/backend/app/services/discord_downloader.py @@ -6,9 +6,11 @@ existing file on disk (`skipped_disk`) instead of fetching it again: //discord//___. -That is FC's gallery-dl config (`gallery_dl.DISCORD_DIRECTORY` / -`DISCORD_FILENAME`) under the per-source base directory -`//`. The name is cleaned the way gallery-dl cleans it +That is what FC's gallery-dl config produced (directory `{channel}`, filename +`{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the +per-source base directory `//`), retired from that +config once Discord moved here; tests/test_discord_naming.py pins the match +against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it on Linux — `/` becomes `_` and control characters are removed, nothing else (`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`, whose Windows set would turn a `:` in a channel or file name into `_` and miss diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 90933bf..4ee68ae 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -95,48 +95,6 @@ BACKFILL_CHUNK_SECONDS = 600 _DEFAULT_GDL_TIMEOUT_SECONDS = 870 -# --- Discord naming --------------------------------------------------------- -# -# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory -# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits -# for an attachment: `channel` is a plain STRING (the channel's name), the -# message is `message_id`, the attachment's position in it is `num`, and there -# is NO `id` key at all. -# -# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as -# "None", so every Discord download since the platform was added landed in a -# directory called `None` as `_None_`. Worse, the sidecar was -# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_ -# sidecar` can never pair with `_None_.png`, so no Discord file ever -# got a Post or a post date, and (b) collides: every `image.png` in a channel -# overwrote the same `image.json`, so the one sidecar that survived described -# whichever message happened to be written last. -# -# The fix names the sidecar EXACTLY like the media minus its extension, so -# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match -# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders -# these patterns against a sanitized copy of the real sidecar, so a key that -# does not exist fails CI instead of silently becoming "None". -DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}" -DISCORD_DIRECTORY = ["{channel}"] - - -def sidecar_name_for(media_pattern: str) -> str | None: - """The metadata filename pattern that names a sidecar exactly like its media. - - Returns None for a pattern that does not end in `.{extension}`, since then - there is no media stem to mirror and the caller must fall back. - """ - suffix = ".{extension}" - if not media_pattern.endswith(suffix): - return None - return media_pattern[: -len(suffix)] + ".json" - - -def metadata_postprocessor(filename: str) -> dict: - return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename} - - def archive_path(images_root: Path) -> Path: """gallery-dl's download archive: the record of what it has already fetched. @@ -425,27 +383,16 @@ class GalleryDLService: # (services/patreon_ingester.py), not gallery-dl. PLATFORM_DEFAULTS = { # subscribestar removed — native-ingester platform now (#71); pixiv - # removed likewise (#129); deviantart removed at #3069 as a dropped - # platform, not a migrated one. The remaining entries are the - # gallery-dl platforms not yet migrated. + # removed likewise (#129); discord likewise (milestone 428, whose + # downloader keeps this config's on-disk naming); deviantart removed at + # #3069 as a dropped platform, not a migrated one. HentaiFoundry is the + # one platform left here, by the operator's choice not to migrate it. "hentaifoundry": { "content_types": ["all"], "directory": [], "filename": "{category}_{index:>03}_{title[:50]}.{extension}", "include": "all", }, - "discord": { - "content_types": ["all"], - "directory": DISCORD_DIRECTORY, - "filename": DISCORD_FILENAME, - # Overrides the global `{filename}.json` sidecar for this extractor - # only — see the Discord naming note above. - "postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))], - "embeds": "all", - "stickers": True, - "reactions": False, - "threads": True, - }, } def __init__( @@ -560,17 +507,6 @@ class GalleryDLService: if source_config.filename_pattern: platform_section["filename"] = source_config.filename_pattern - # A platform that names its sidecar after its media must keep doing so - # under a per-source filename override, or the pairing breaks exactly the - # way Discord's did. No metadata wanted means no platform postprocessor - # either — the global list was already dropped above. - if "postprocessors" in platform_section: - mirrored = sidecar_name_for(platform_section.get("filename") or "") - if not source_config.save_metadata or mirrored is None: - platform_section.pop("postprocessors") - else: - platform_section["postprocessors"] = [metadata_postprocessor(mirrored)] - platform_section["metadata"] = source_config.save_metadata return config @@ -818,9 +754,6 @@ class GalleryDLService: if cookies_path: config["extractor"]["cookies"] = cookies_path - if auth_token and platform == "discord": - config["extractor"].setdefault("discord", {}) - config["extractor"]["discord"]["token"] = auth_token with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False, dir=str(self._config_dir), @@ -1004,8 +937,6 @@ class GalleryDLService: config = self._build_config_for_source(platform, source_config, artist_slug) if cookies_path: config["extractor"]["cookies"] = cookies_path - if auth_token and platform == "discord": - config["extractor"].setdefault("discord", {})["token"] = auth_token with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False, dir=str(self._config_dir), diff --git a/backend/app/services/scheduler_service.py b/backend/app/services/scheduler_service.py index 43187cf..b24a6fe 100644 --- a/backend/app/services/scheduler_service.py +++ b/backend/app/services/scheduler_service.py @@ -115,6 +115,32 @@ async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime return active +def backfill_ready(source: Source) -> bool: + """A deep walk the operator started, with budget left and no failure + backing it off — due NOW rather than at its next scheduled check. + + A backfill runs one time-boxed chunk per download (plan #693), and nothing + queued the next chunk: each waited for the source's regular interval. At + the 8-hour default a freshly armed backfill sat untouched until the next + check (the operator armed one on 2026-09-25 and saw nothing happen) and a + five-chunk walk took most of two days. The tick's in-flight guard keeps + one chunk at a time per source and the platform lock one walk per + platform, so "due every tick" means "next chunk as soon as the last one + ends". + + The failure gate is what keeps a broken source from retrying every + minute: any failed chunk raises `consecutive_failures`, which drops the + source back onto its backed-off interval. A chunk that fails to progress + twice marks the walk stalled (download_service), which ends it here too. + """ + co = source.config_overrides or {} + return ( + co.get("_backfill_state") == "running" + and (source.backfill_runs_remaining or 0) > 0 + and not (source.consecutive_failures or 0) + ) + + async def select_due_sources(session: AsyncSession) -> list[Source]: """Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval. @@ -123,6 +149,9 @@ async def select_due_sources(session: AsyncSession) -> list[Source]: cooldown is the preventive half of the burst-prevention pair (per-source consecutive_failures backoff handles the offending source itself). + A running backfill (`backfill_ready`) is due on every tick, and whether + or not its artist is on auto-check — the operator started it by hand. + Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked sources go first, then the longest-since-checked, so the most overdue sources hit Celery's FIFO download queue first. Anti-starvation: if @@ -135,7 +164,6 @@ async def select_due_sources(session: AsyncSession) -> list[Source]: .options(selectinload(Source.artist)) .join(Artist, Source.artist_id == Artist.id) .where(Source.enabled.is_(True)) - .where(Artist.auto_check.is_(True)) .order_by(Source.last_checked_at.asc().nulls_first(), Source.id) )).scalars().all() @@ -147,6 +175,11 @@ async def select_due_sources(session: AsyncSession) -> list[Source]: for s in rows: if s.platform in cooldowns: continue + if backfill_ready(s): + due.append(s) + continue + if not s.artist.auto_check: + continue interval = compute_effective_interval(s, s.artist, settings) if s.last_checked_at is None: due.append(s) @@ -161,6 +194,8 @@ def compute_next_check_at( source: Source, artist: Artist, settings: ImportSettings, ) -> datetime | None: """Return the projected datetime of the next check, or None if never checked.""" + if backfill_ready(source): + return datetime.now(UTC) if source.last_checked_at is None: return None interval = compute_effective_interval(source, artist, settings) diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 439d566..42201a7 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -18,6 +18,7 @@ from ..models import ( Source, ) from .db_helpers import failing_sources_clause +from .download_backends import uses_native_ingester from .gallery_dl import ErrorType from .membership_reconcile import KEPT_KEY, STOPPED_KEY from .membership_roster import gated_reasons_for_sources @@ -125,6 +126,11 @@ class SourceRecord: "backfill_posts": self.backfill_posts, "tier_gated_count": self.tier_gated_count, "gated_reason": self.gated_reason, + # Recover / recapture exist only on the native ingester. Sent so the + # UI asks the backend's own predicate instead of keeping a copy of + # the platform list — the copy said "patreon, subscribestar" for a + # day after Discord went native (milestone 428). + "native_ingester": uses_native_ingester(self.platform), } @@ -551,8 +557,8 @@ class SourceService: whole source); the two flags are mutually exclusive, so arming recapture clears bypass_seen. Clears prior cursor/chunk/stall state so it walks fresh from the top. The flag is cleared on completion (download_service) - and on stop. Recapture is Patreon-only (the native ingester's post-record - capture); inert elsewhere. The UI gates the action to Patreon sources.""" + and on stop. Recapture needs the native ingester's post-record capture, + so the UI offers it on native sources only (`native_ingester`).""" source = (await self.session.execute( select(Source).where(Source.id == source_id) )).scalar_one_or_none() diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index 34e484f..3512b70 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -76,10 +76,10 @@ const running = computed(() => props.source.backfill_state === 'running') const recovering = computed(() => !!props.source.backfill_bypass_seen) const recapturing = computed(() => !!props.source.backfill_recapture) // Recover / recapture are native-ingester features (ledger-bypass re-walk and -// post-text re-grab), available to every native platform — not just Patreon. -// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS. -const NATIVE_PLATFORMS = ['patreon', 'subscribestar'] -const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform)) +// post-text re-grab), available on every native platform. The backend says +// which those are (`native_ingester`); a copied list here went stale when +// Discord moved over. +const isNative = computed(() => !!props.source.native_ingester)