diff --git a/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py b/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py new file mode 100644 index 0000000..c37e499 --- /dev/null +++ b/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py @@ -0,0 +1,145 @@ +"""nullable post.source_id + denormalized post.artist_id; retire sidecar synthetics + +Revision ID: 0030 +Revises: 0029 +Create Date: 2026-06-01 + +Operator-asked 2026-06-01 after the Dymkens orphan investigation: the +sidecar synthetic Source pattern (`sidecar::` rows +with enabled=false) was technically correct but misled the operator +into thinking they had phantom subscriptions. The synthetics existed +solely to satisfy `Post.source_id NOT NULL` for filesystem-imported +content with no real subscription. + +This migration makes the data model honest: + +1. **Post gets a denormalized `artist_id` column** so artist filters + work without traversing `Post → Source.artist_id`. Backfilled from + the existing Source linkage, then NOT NULL'd. +2. **`Post.source_id` becomes nullable**, FK ondelete `CASCADE` → `SET + NULL`. Deleting a Source detaches its Posts instead of destroying + imported content (semantically: subscription ends, archive stays). +3. **`ImageProvenance.source_id` becomes nullable** with the same FK + semantic change. +4. **Sidecar synthetic Sources are deleted** — first NULL out the + FKs from Post + ImageProvenance pointing at them (so the implicit + CASCADE doesn't fire), then delete. DownloadEvent FK is unchanged + (still CASCADE'd, NOT NULL'd) — synthetics have `enabled=false` + so no events exist for them. + +Uniqueness handling: the existing `uq_post_source_external_id` +(source_id, external_post_id) keeps working for source-bound Posts +(Postgres treats NULL != NULL so NULL-source rows aren't deduped by +it). A second partial unique index covers the NULL-source case on +(artist_id, external_post_id) so filesystem-imported posts still +dedupe within an artist. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import text + +revision: str = "0030" +down_revision: Union[str, None] = "0029" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + conn = op.get_bind() + + # Step 1: add Post.artist_id, initially nullable for backfill. + # FK naming follows the Base.metadata naming_convention + # (fk___) — alembic 0001 set this up. + op.add_column( + "post", + sa.Column("artist_id", sa.Integer, nullable=True), + ) + op.create_foreign_key( + "fk_post_artist_id_artist", "post", "artist", + ["artist_id"], ["id"], ondelete="CASCADE", + ) + + # Step 2: backfill from Source.artist_id (every existing Post has a + # Source today, so every row gets populated). + conn.execute(text(""" + UPDATE post p + SET artist_id = s.artist_id + FROM source s + WHERE p.source_id = s.id AND p.artist_id IS NULL + """)) + + # Sanity: count any remaining NULLs. Should be zero pre-this-migration. + remaining = conn.execute(text( + "SELECT COUNT(*) FROM post WHERE artist_id IS NULL" + )).scalar_one() + if remaining: + raise RuntimeError( + f"alembic 0030: {remaining} post rows have no resolvable " + f"artist_id after backfill. Investigate before continuing." + ) + + # Step 3: enforce NOT NULL + add index for artist-filter queries. + op.alter_column("post", "artist_id", nullable=False) + op.create_index("ix_post_artist_id", "post", ["artist_id"]) + + # Step 4: relax post.source_id + flip FK to SET NULL. The original FK + # name from alembic 0001 is `fk_post_source_id_source` per the + # NAMING_CONVENTION in models/base.py. + op.alter_column("post", "source_id", nullable=True) + op.drop_constraint("fk_post_source_id_source", "post", type_="foreignkey") + op.create_foreign_key( + "fk_post_source_id_source", "post", "source", + ["source_id"], ["id"], ondelete="SET NULL", + ) + + # Step 5: relax image_provenance.source_id + flip FK to SET NULL. + op.alter_column("image_provenance", "source_id", nullable=True) + op.drop_constraint( + "fk_image_provenance_source_id_source", "image_provenance", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_image_provenance_source_id_source", "image_provenance", "source", + ["source_id"], ["id"], ondelete="SET NULL", + ) + + # Step 6: partial unique index on (artist_id, external_post_id) for + # NULL-source Posts. The existing uq_post_source_external_id keeps + # guarding source-bound rows; NULL-source rows now dedupe within + # an artist. + op.execute( + "CREATE UNIQUE INDEX uq_post_artist_external_id_null_source " + "ON post (artist_id, external_post_id) " + "WHERE source_id IS NULL" + ) + + # Step 7: retire sidecar synthetic Sources. NULL out the references + # FIRST (the new FK is SET NULL so CASCADE wouldn't fire anyway, but + # being explicit makes the intent clear). Then delete the synthetic + # source rows. Any DownloadEvent rows under synthetics CASCADE-die + # with the source — synthetics have enabled=false so there shouldn't + # be any in practice. + conn.execute(text(""" + UPDATE post + SET source_id = NULL + WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%') + """)) + conn.execute(text(""" + UPDATE image_provenance + SET source_id = NULL + WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%') + """)) + deleted = conn.execute(text( + "DELETE FROM source WHERE url LIKE 'sidecar:%' RETURNING id" + )).rowcount + print(f"alembic 0030: deleted {deleted} sidecar synthetic source rows") + + +def downgrade() -> None: + # Lossy migration — the deleted sidecar synthetics can't be + # restored from the orphan post.source_id / image_provenance.source_id + # values, and the partial unique index encodes a constraint that + # NULL-source Posts may now exist. No safe downgrade. + pass diff --git a/alembic/versions/0031_source_backfill_runs_remaining.py b/alembic/versions/0031_source_backfill_runs_remaining.py new file mode 100644 index 0000000..ed140fc --- /dev/null +++ b/alembic/versions/0031_source_backfill_runs_remaining.py @@ -0,0 +1,45 @@ +"""source.backfill_runs_remaining: sticky deep-scan mode + +Revision ID: 0031 +Revises: 0030 +Create Date: 2026-06-01 + +Tick vs backfill mode for subscription downloads. When +`backfill_runs_remaining > 0`, the next N download runs use +`skip: True` + 30-min timeout (walk full history). When 0, runs use +`skip: "exit:20"` + 14.5-min timeout (catch-up mode, exits early once +20 contiguous archived items are seen). + +Operator-flagged 2026-06-01 (Knuxy run #38887): a creator with ~550 +archived posts saturates the 870s catch-up timeout even when there is +no new content, because gallery-dl's default `skip: True` keeps walking. +Tick mode short-circuits that; backfill mode is the explicit opt-in for +deep history scans. + +Default 0 (all existing subscriptions start in tick mode). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0031" +down_revision: Union[str, None] = "0030" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "source", + sa.Column( + "backfill_runs_remaining", + sa.Integer, + nullable=False, + server_default="0", + ), + ) + + +def downgrade() -> None: + op.drop_column("source", "backfill_runs_remaining") diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index a1312ba..fe234ee 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -120,6 +120,31 @@ async def delete_source(source_id: int): return "", 204 +@sources_bp.route("//backfill", methods=["POST"]) +async def set_backfill(source_id: int): + """Plan #544: arm a source for backfill mode for the next N download + runs. Body: `{"runs": int}` (1..10, default 3). Returns the updated + source dict. While backfill_runs_remaining > 0, downloads use + gallery-dl's full-walk config (skip: True + 30-min timeout) instead + of the catch-up default (skip: "exit:20" + 14.5-min timeout).""" + payload = await request.get_json(silent=True) or {} + runs = payload.get("runs", 3) + try: + runs = int(runs) + except (TypeError, ValueError): + return _bad("invalid_runs", detail="runs must be an integer") + async with get_session() as session: + try: + record = await SourceService(session).set_backfill_runs( + source_id, runs, + ) + except LookupError: + return _bad("not_found", status=404) + except ValueError as exc: + return _bad("invalid_runs", detail=str(exc)) + return jsonify(record.to_dict()) + + @sources_bp.route("//check", methods=["POST"]) async def check_source(source_id: int): """FC-3c: enqueue a download for this source. diff --git a/backend/app/models/image_provenance.py b/backend/app/models/image_provenance.py index ec3e756..eaa47df 100644 --- a/backend/app/models/image_provenance.py +++ b/backend/app/models/image_provenance.py @@ -34,8 +34,12 @@ class ImageProvenance(Base): post_id: Mapped[int] = mapped_column( ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True ) - source_id: Mapped[int] = mapped_column( - ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + # Nullable since alembic 0030 — provenance rows for filesystem-imported + # content with no subscription have NULL source_id. FK ondelete SET + # NULL so deleting a Source detaches its provenance rows instead of + # destroying the linkage between image and post. + source_id: Mapped[int | None] = mapped_column( + ForeignKey("source.id", ondelete="SET NULL"), nullable=True, index=True ) captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) captured_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/models/post.py b/backend/app/models/post.py index d5263c9..d9830a2 100644 --- a/backend/app/models/post.py +++ b/backend/app/models/post.py @@ -1,6 +1,9 @@ -"""Post — provenance anchor for content downloaded from a Source. +"""Post — provenance anchor for one creator post (may contain many images). -A Post is one creator post; it may contain many images/videos. +`source_id` is nullable since alembic 0030 — filesystem-imported posts +with no live subscription have NULL source_id. `artist_id` is the +denormalized always-present link to the creator (added in 0030 so +artist-filter queries don't depend on the Source detour). """ from datetime import datetime @@ -14,12 +17,25 @@ from .base import Base class Post(Base): __tablename__ = "post" __table_args__ = ( + # Source-bound dedup. Postgres treats NULL != NULL so rows + # with source_id IS NULL aren't deduped by this constraint; + # the partial unique index `uq_post_artist_external_id_null_source` + # (created in alembic 0030) covers that case via + # (artist_id, external_post_id). UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) - source_id: Mapped[int] = mapped_column( - ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + source_id: Mapped[int | None] = mapped_column( + ForeignKey("source.id", ondelete="SET NULL"), nullable=True, index=True + ) + # Denormalized; always equals source.artist_id when source_id is set + # (the importer is responsible for keeping them consistent on insert). + # Filter queries (artist detail, artist-scoped posts feed) use this + # directly instead of joining through Source. + artist_id: Mapped[int] = mapped_column( + ForeignKey("artist.id", ondelete="CASCADE"), + nullable=False, index=True, ) external_post_id: Mapped[str] = mapped_column(String(128), nullable=False) post_url: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 9c90e81..407227e 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -29,4 +29,13 @@ class Source(Base): check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True) consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # alembic 0031: sticky deep-scan budget. When > 0, the next N download + # runs use gallery-dl's full-walk config (skip: True + 1800s timeout); + # when 0, runs use tick mode (skip: "exit:20" + 870s, exits early once + # 20 contiguous archived items are seen). Auto-decrements per run, with + # an auto-reset to 0 on clean exit + zero downloads (queue drained). + backfill_runs_remaining: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0", + ) + artist = relationship("Artist", back_populates="sources") diff --git a/backend/app/services/artist_service.py b/backend/app/services/artist_service.py index 56f19ff..0947411 100644 --- a/backend/app/services/artist_service.py +++ b/backend/app/services/artist_service.py @@ -58,13 +58,17 @@ class ArtistService: ) ).scalar_one() + # Posts under this artist that have at least one image attached. + # Use Post.artist_id (alembic 0030) for the artist filter; keep + # the ImageProvenance JOIN so date bounds reflect only image- + # bearing posts (matches the original semantic). NULL-source + # posts now surface too. date_row = ( await self.session.execute( select(func.min(Post.post_date), func.max(Post.post_date)) .select_from(Post) .join(ImageProvenance, ImageProvenance.post_id == Post.id) - .join(Source, Source.id == ImageProvenance.source_id) - .where(Source.artist_id == aid) + .where(Post.artist_id == aid) ) ).first() dmin, dmax = date_row if date_row else (None, None) @@ -98,14 +102,14 @@ class ArtistService: ) ).all() + # Same Post.artist_id direct filter — counts NULL-source posts too. month = func.date_trunc("month", Post.post_date).label("m") activity = ( await self.session.execute( select(month, func.count(func.distinct(ImageProvenance.image_record_id))) .select_from(Post) .join(ImageProvenance, ImageProvenance.post_id == Post.id) - .join(Source, Source.id == ImageProvenance.source_id) - .where(and_(Source.artist_id == aid, Post.post_date.isnot(None))) + .where(and_(Post.artist_id == aid, Post.post_date.isnot(None))) .group_by(month) .order_by(month) ) @@ -114,9 +118,7 @@ class ArtistService: post_count = ( await self.session.execute( select(func.count(func.distinct(Post.id))) - .select_from(Post) - .join(Source, Source.id == Post.source_id) - .where(Source.artist_id == aid) + .where(Post.artist_id == aid) ) ).scalar_one() diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 6e917c3..529da83 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -25,7 +25,14 @@ from sqlalchemy.orm import joinedload from ..models import Artist, DownloadEvent, Source from .credential_service import CredentialService -from .gallery_dl import GalleryDLService, SourceConfig +from .gallery_dl import ( + BACKFILL_SKIP_VALUE, + BACKFILL_TIMEOUT_SECONDS, + TICK_SKIP_VALUE, + ErrorType, + GalleryDLService, + SourceConfig, +) from .importer import Importer from .patreon_resolver import resolve_campaign_id from .scheduler_service import set_platform_cooldown @@ -84,6 +91,18 @@ class DownloadService: ctx = setup source_config = SourceConfig.from_dict(ctx["config_overrides"] or {}) + # alembic 0031 / plan #544: derive skip_value + timeout from the + # source's backfill_runs_remaining counter. When > 0, walk the full + # post history (skip: True + 1800s); when 0, exit gallery-dl after + # 20 contiguous archived items (skip: "exit:20" + the default + # 870s). Operator sets backfill via POST /api/sources/{id}/backfill. + backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0 + if backfill_remaining > 0: + skip_value: bool | str = BACKFILL_SKIP_VALUE + source_config.timeout = BACKFILL_TIMEOUT_SECONDS + else: + skip_value = TICK_SKIP_VALUE + effective_url = _effective_url( ctx["platform"], ctx["url"], ctx["config_overrides"] or {} ) @@ -95,6 +114,7 @@ class DownloadService: source_config=source_config, cookies_path=ctx["cookies_path"], auth_token=ctx["auth_token"], + skip_value=skip_value, ) resolved_campaign_id: str | None = None @@ -121,6 +141,7 @@ class DownloadService: source_config=source_config, cookies_path=ctx["cookies_path"], auth_token=ctx["auth_token"], + skip_value=skip_value, ) return await self._phase3_persist( @@ -185,6 +206,7 @@ class DownloadService: "config_overrides": dict(source.config_overrides or {}), "cookies_path": cookies_path, "auth_token": auth_token, + "backfill_runs_remaining": source.backfill_runs_remaining or 0, } async def _phase3_persist( @@ -259,12 +281,21 @@ class DownloadService: run_stats["quarantined_count"] = dl_result.files_quarantined stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr) - status = "ok" if (dl_result.success and import_summary["errors"] == 0) else "error" + # Plan #544: PARTIAL means the run downloaded ≥1 file but the + # subprocess didn't finish in budget (typically wall-clock timeout + # mid-walk). Real work happened; the next tick continues via + # gallery-dl's archive. NOT a failure for status purposes. + if dl_result.success and import_summary["errors"] == 0: + status = "ok" + elif dl_result.error_type == ErrorType.PARTIAL and import_summary["errors"] == 0: + status = "ok" + else: + status = "error" ev.status = status ev.finished_at = datetime.now(UTC) ev.files_count = import_summary["attached"] ev.bytes_downloaded = bytes_downloaded - ev.error = dl_result.error_message if not dl_result.success else None + ev.error = dl_result.error_message if status == "error" else None ev.metadata_ = { "run_stats": run_stats, "error_type": dl_result.error_type.value if dl_result.error_type else None, @@ -279,6 +310,19 @@ class DownloadService: source_id=ctx["source_id"], status=status, error_message=ev.error, error_type=dl_result.error_type.value if dl_result.error_type else None, ) + # Plan #544: backfill lifecycle — auto-complete when a clean + # backfill run drained the queue (gallery-dl exited 0 + zero files + # downloaded means there was nothing to fetch); otherwise decrement + # the counter. Next tick falls back to tick mode once it hits 0. + backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0 + if backfill_remaining > 0: + src = (await self.async_session.execute( + select(Source).where(Source.id == ctx["source_id"]) + )).scalar_one() + if dl_result.return_code == 0 and dl_result.files_downloaded == 0: + src.backfill_runs_remaining = 0 + else: + src.backfill_runs_remaining = max(0, backfill_remaining - 1) await self.async_session.commit() return event_id diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 3e19a69..0f19793 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -39,9 +39,29 @@ class ErrorType(StrEnum): HTTP_ERROR = "http_error" UNSUPPORTED_URL = "unsupported_url" VALIDATION_FAILED = "validation_failed" + # Run made real progress (downloaded ≥1 file) but did not finish in the + # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status + # mapping classifies this as "ok" because the next tick continues. + PARTIAL = "partial" UNKNOWN_ERROR = "unknown_error" +# Tick mode (routine cron polls): skip ≤20 contiguous already-archived +# items, then exit gallery-dl. Established subscription with zero new +# content exits in ~30s of HEAD requests instead of walking to the bottom +# of the post history (which can be hours for prolific creators). 20 (not +# 5) is operator-set headroom against any edge case where paywalled or +# otherwise-non-downloadable items might interleave with archived ones — +# 20 contiguous HEADs is still negligible. +TICK_SKIP_VALUE = "exit:20" + +# Backfill mode (operator-triggered deep scan): walk the full history. +# Source.backfill_runs_remaining > 0 selects this mode; the longer +# timeout below absorbs creators with thousands of posts. +BACKFILL_SKIP_VALUE = True +BACKFILL_TIMEOUT_SECONDS = 1800 + + # 30 seconds shy of download_source's Celery soft_time_limit (900s, see # tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before # Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race, @@ -56,12 +76,19 @@ _DEFAULT_GDL_TIMEOUT_SECONDS = 870 @dataclass class SourceConfig: + """Per-source overrides loaded from Source.config_overrides JSON. + + Note: the gallery-dl `skip` value (tick vs backfill, see TICK_SKIP_VALUE / + BACKFILL_SKIP_VALUE) is NOT carried here — it derives from the + Source.backfill_runs_remaining column at the download_service layer + and is passed to _build_config_for_source as `skip_value`. Same for + the per-run subprocess timeout. + """ content_types: list[str] = field(default_factory=lambda: ["all"]) sleep: float | None = None sleep_request: float | None = None directory_pattern: str | None = None filename_pattern: str | None = None - skip_existing: bool = True save_metadata: bool = True timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS @@ -73,7 +100,6 @@ class SourceConfig: sleep_request=data.get("sleep_request"), directory_pattern=data.get("directory_pattern"), filename_pattern=data.get("filename_pattern"), - skip_existing=data.get("skip_existing", True), save_metadata=data.get("save_metadata", True), timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS), ) @@ -231,6 +257,25 @@ class GalleryDLService: "part-directory": str(self._config_dir / "temp"), "retries": 3, "timeout": 120.0, + # Forward Patreon as Referer/Origin to yt-dlp when it + # fetches video manifests. Operator-flagged 2026-06-01 + # (DaferQ patreon): video posts hosted on Mux carry a JWT + # playback restriction that checks Referer/Origin on every + # request — not just the token signature. gallery-dl's + # HEAD probe to stream.mux.com returns 200 (the token is + # valid), but yt-dlp's actual GET-with-Range to fetch the + # m3u8 manifest 403s because yt-dlp sends its own default + # Referer, which Mux's policy rejects. Forcing the right + # headers fixes the headers-only case; Mux IP-range + # restrictions are unfixable from here. + "ytdl": { + "raw-options": { + "http_headers": { + "Referer": "https://www.patreon.com/", + "Origin": "https://www.patreon.com", + }, + }, + }, }, "output": {"progress": True}, } @@ -245,7 +290,18 @@ class GalleryDLService: platform: str, source_config: SourceConfig, artist_slug: str, + skip_value: bool | str = BACKFILL_SKIP_VALUE, ) -> dict: + """`skip_value` controls gallery-dl's archive-walk behavior: + - True (BACKFILL_SKIP_VALUE): walk full post history, skipping + archived items but continuing past them. Used in backfill mode. + - "exit:20" (TICK_SKIP_VALUE): exit gallery-dl after 20 + contiguous archived items. Used in tick (routine catch-up) + mode for fast no-op syncs on creators with deep history. + - False: don't skip — redownload everything (not used in FC). + The caller (download_service) chooses based on + Source.backfill_runs_remaining. + """ config = json.loads(json.dumps(self._get_default_config())) # deep copy destination = str(self.images_root / artist_slug / platform) @@ -255,7 +311,7 @@ class GalleryDLService: config["extractor"]["sleep"] = source_config.sleep if source_config.sleep_request is not None: config["extractor"]["sleep-request"] = source_config.sleep_request - config["extractor"]["skip"] = source_config.skip_existing + config["extractor"]["skip"] = skip_value if source_config.save_metadata: config["extractor"]["postprocessors"] = [ @@ -394,6 +450,22 @@ class GalleryDLService: f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}", ) + # Partial-success: the subprocess exited non-zero (typically because + # the wall-clock timeout fired mid-walk), but it had downloaded ≥1 + # file by then and no source-level error category fired. The work + # the run DID do is real; gallery-dl's archive will pick up where + # it left off on the next tick. Mapped to status="ok" downstream + # (download_service.py) so this doesn't flag the source as + # "needs attention." Operator-flagged 2026-06-01 after a Knuxy + # patreon run downloaded hundreds of files then ran red on timeout. + files_downloaded = self._count_downloaded_files(stdout) + if not has_actual_error and files_downloaded > 0: + return ( + ErrorType.PARTIAL, + f"Downloaded {files_downloaded} file{'s' if files_downloaded != 1 else ''}; " + "run did not complete in budget — next tick will continue", + ) + return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" def _count_downloaded_files(self, stdout: str) -> int: @@ -545,6 +617,7 @@ class GalleryDLService: source_config: SourceConfig | None = None, cookies_path: str | None = None, auth_token: str | None = None, + skip_value: bool | str = BACKFILL_SKIP_VALUE, ) -> DownloadResult: start_time = time.time() started_at = datetime.now(UTC).isoformat() @@ -552,7 +625,9 @@ class GalleryDLService: if source_config is None: source_config = SourceConfig() - config = self._build_config_for_source(platform, source_config, artist_slug) + config = self._build_config_for_source( + platform, source_config, artist_slug, skip_value=skip_value, + ) if cookies_path: config["extractor"]["cookies"] = cookies_path diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 095890d..3331510 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -20,8 +20,9 @@ from datetime import datetime from sqlalchemy import Select, and_, exists, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import aliased -from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag +from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag from ..models.tag import image_tag CURSOR_SEPARATOR = "|" @@ -133,10 +134,23 @@ def _provenance_clause(post_id, artist_id): ImageProvenance.post_id == post_id, ) if artist_id is not None: + # Use Post.artist_id (alembic 0030 denormalized column) instead + # of joining through ImageProvenance.source_id → Source.artist_id. + # The denormalization is the always-present linkage; the source + # path now drops NULL-source provenance rows (filesystem-imported + # content) which would otherwise vanish from artist-filtered + # gallery views. + # ALIAS Post: the gallery query outer-joins Post on + # ImageRecord.primary_post_id (`_outer_join_primary_post`). + # SQLAlchemy would otherwise correlate a bare `Post` reference + # in this EXISTS subquery to that outer Post (which is NULL for + # images with no primary post), and the filter would silently + # match nothing. + post_inner = aliased(Post) return exists().where( ImageProvenance.image_record_id == ImageRecord.id, - ImageProvenance.source_id == Source.id, - Source.artist_id == artist_id, + ImageProvenance.post_id == post_inner.id, + post_inner.artist_id == artist_id, ) return None diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index c66caa8..6e3ed23 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -212,10 +212,10 @@ class Importer: which would lose the surrounding scan's progress — and re-run `stmt` (scalar_one) to return the row the other worker created. - Centralizes the pattern shared by _find_or_create_source, - _source_for_sidecar, and _find_or_create_post. The plain - SELECT-then-INSERT version lost races under the 5-min recovery sweep - (operator-flagged 2026-05-26).""" + Centralizes the pattern shared by _find_or_create_source and + _find_or_create_post. The plain SELECT-then-INSERT version lost + races under the 5-min recovery sweep (operator-flagged + 2026-05-26).""" existing = self.session.execute(stmt).scalar_one_or_none() if existing is not None: return existing @@ -258,47 +258,25 @@ class Importer: lambda: Source(artist_id=artist_id, platform=platform, url=url), ) - def _source_for_sidecar( - self, *, artist_id: int, platform: str, artist_slug: str, - ) -> Source: - """Sidecar-import Source resolver. Used by both filesystem imports - and gallery-dl downloads (both write sidecar JSON, both flow through - _apply_sidecar / _capture_attachment). + def _lookup_source_for_sidecar( + self, *, artist_id: int, platform: str, + ) -> Source | None: + """Find the real subscription Source for (artist, platform), or + None if no subscription exists. - Source represents a subscription feed (one per artist+platform — the - URL polled by the FC-3 downloader). The filesystem importer used to - call _find_or_create_source(url=sd.post_url), creating one Source - row per post URL — 100s of junk Sources per artist, all with - enabled=True, polluting the artist detail page and tricking the - subscription checker into trying to poll patreon post URLs as feeds. - Operator-flagged 2026-05-26; consolidated via alembic 0022. - - Resolution order: prefer a real (non-sidecar) Source over a - synthetic anchor. When alembic 0022 ran, it may have rewritten - per-post Sources into `sidecar::` synthetic - anchors. If the operator later added the real subscription, both - rows now coexist. A naive `ORDER BY id ASC LIMIT 1` lookup would - pick the older synthetic and silently attach every gallery-dl - download to the wrong Source — operator-flagged 2026-05-31 after - the Subscriptions UI surfaced the phantom anchors. Pick the real - one when one exists; fall back to the synthetic; only create a - new synthetic when nothing exists for (artist, platform). + Pre-alembic-0030 this method would CREATE a synthetic + `sidecar::` Source when no real one existed — + because `Post.source_id` was NOT NULL and the importer needed + something to attach Posts to. Alembic 0030 relaxed both + `Post.source_id` and `ImageProvenance.source_id` to nullable, so + synthetic anchors are obsolete; the importer now leaves + source_id as None when no subscription exists for the (artist, + platform). Operator-asked 2026-06-01: synthetic Sources had + leaked into the Subscriptions UI as phantom subscriptions and + the operator wanted the data model to truthfully say "this + content has no live subscription." """ - real_stmt = ( - select(Source) - .where( - Source.artist_id == artist_id, - Source.platform == platform, - ~Source.url.like("sidecar:%"), - ) - .order_by(Source.id.asc()) - .limit(1) - ) - real = self.session.execute(real_stmt).scalar_one_or_none() - if real is not None: - return real - - any_stmt = ( + stmt = ( select(Source) .where( Source.artist_id == artist_id, @@ -307,29 +285,37 @@ class Importer: .order_by(Source.id.asc()) .limit(1) ) - return self._get_or_create( - any_stmt, - lambda: Source( - artist_id=artist_id, - platform=platform, - url=f"sidecar:{platform}:{artist_slug}", - enabled=False, - ), - ) + return self.session.execute(stmt).scalar_one_or_none() def _find_or_create_post( - self, *, source_id: int, external_post_id: str, + self, *, source_id: int | None, external_post_id: str, + artist_id: int, ) -> Post: - """Race-safe find-or-create on `post` keyed by - (source_id, external_post_id). Mirrors `_find_or_create_source` - — same savepoint + IntegrityError-recovery pattern.""" - stmt = select(Post).where( - Post.source_id == source_id, - Post.external_post_id == external_post_id, - ) + """Race-safe find-or-create on `post`. Keyed by + (source_id, external_post_id) when source_id is set — the + `uq_post_source_external_id` constraint guards. For NULL-source + posts the existence check matches on (artist_id, external_post_id), + which the partial unique index `uq_post_artist_external_id_null_source` + (alembic 0030) guards. Same savepoint + IntegrityError-recovery + pattern as the rest of the helpers.""" + if source_id is not None: + stmt = select(Post).where( + Post.source_id == source_id, + Post.external_post_id == external_post_id, + ) + else: + stmt = select(Post).where( + Post.source_id.is_(None), + Post.artist_id == artist_id, + Post.external_post_id == external_post_id, + ) return self._get_or_create( stmt, - lambda: Post(source_id=source_id, external_post_id=external_post_id), + lambda: Post( + source_id=source_id, + artist_id=artist_id, + external_post_id=external_post_id, + ), ) def import_one(self, source: Path) -> ImportResult: @@ -370,12 +356,14 @@ class Importer: return None sd = parse_sidecar(data) platform = sd.platform or "unknown" - src = self._source_for_sidecar( - artist_id=artist.id, platform=platform, artist_slug=artist.slug, + src = self._lookup_source_for_sidecar( + artist_id=artist.id, platform=platform, ) epid = sd.external_post_id or sc.stem return self._find_or_create_post( - source_id=src.id, external_post_id=epid, + source_id=src.id if src else None, + external_post_id=epid, + artist_id=artist.id, ) def _capture_attachment( @@ -858,14 +846,15 @@ class Importer: src = explicit_source else: platform = sd.platform or "unknown" - src = self._source_for_sidecar( + src = self._lookup_source_for_sidecar( artist_id=artist.id, platform=platform, - artist_slug=artist.slug, ) epid = sd.external_post_id or sc.stem post = self._find_or_create_post( - source_id=src.id, external_post_id=epid, + source_id=src.id if src else None, + external_post_id=epid, + artist_id=artist.id, ) if sd.post_url is not None: post.post_url = sd.post_url @@ -901,7 +890,7 @@ class Importer: ImageProvenance( image_record_id=record.id, post_id=post.id, - source_id=src.id, + source_id=src.id if src else None, captured_metadata=sd.raw, ) ) diff --git a/backend/app/services/platforms/base.py b/backend/app/services/platforms/base.py index 8a626b7..737b529 100644 --- a/backend/app/services/platforms/base.py +++ b/backend/app/services/platforms/base.py @@ -96,10 +96,12 @@ def str_field(v) -> str | None: # Shared gallery-dl invocation defaults. Embedded in each platform's # default_config (with platform-specific overrides) so per-platform # choices stay explicit. +# Note: the gallery-dl `skip` value (tick "exit:20" vs backfill True) is +# NOT here — it's derived from Source.backfill_runs_remaining at download +# time. See plan #544 / gallery_dl.TICK_SKIP_VALUE,BACKFILL_SKIP_VALUE. GD_DEFAULTS = { "sleep": 3.0, "sleep_request": 1.5, - "skip_existing": True, "save_metadata": True, "timeout": 3600, } diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index 634cfd3..66d2745 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -69,13 +69,19 @@ class PostFeedService: raise ValueError("direction must be 'older' or 'newer'") sort_key = _sort_key() + # Artist via the denormalized Post.artist_id (alembic 0030); + # Source via LEFT JOIN since post.source_id can now be NULL for + # filesystem-imported posts with no live subscription. A + # platform= filter implicitly excludes NULL-source posts (they + # have no platform); an artist_id= filter still surfaces them + # because Post.artist_id is always set. stmt = ( select(Post, Artist, Source) - .join(Source, Post.source_id == Source.id) - .join(Artist, Source.artist_id == Artist.id) + .join(Artist, Post.artist_id == Artist.id) + .outerjoin(Source, Post.source_id == Source.id) ) if artist_id is not None: - stmt = stmt.where(Source.artist_id == artist_id) + stmt = stmt.where(Post.artist_id == artist_id) if platform is not None: stmt = stmt.where(Source.platform == platform) if cursor: @@ -135,8 +141,8 @@ class PostFeedService: cursor for each end. Returns None if the post doesn't exist.""" anchor = (await self.session.execute( select(Post, Artist, Source) - .join(Source, Post.source_id == Source.id) - .join(Artist, Source.artist_id == Artist.id) + .join(Artist, Post.artist_id == Artist.id) + .outerjoin(Source, Post.source_id == Source.id) .where(Post.id == post_id) )).one_or_none() if anchor is None: @@ -168,8 +174,8 @@ class PostFeedService: async def get_post(self, post_id: int) -> dict | None: row = (await self.session.execute( select(Post, Artist, Source) - .join(Source, Post.source_id == Source.id) - .join(Artist, Source.artist_id == Artist.id) + .join(Artist, Post.artist_id == Artist.id) + .outerjoin(Source, Post.source_id == Source.id) .where(Post.id == post_id) )).one_or_none() if row is None: @@ -260,7 +266,7 @@ class PostFeedService: return out def _to_dict( - self, post: Post, artist: Artist, source: Source, + self, post: Post, artist: Artist, source: Source | None, thumbs_map: dict, atts_map: dict, ) -> dict: plain_full = html_to_plain(post.description) if post.description else None @@ -269,6 +275,9 @@ class PostFeedService: else: description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT) thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0}) + # `source` is null for filesystem-imported posts with no live + # subscription (alembic 0030). Frontend renders that as a + # "filesystem import" affordance instead of a platform chip. return { "id": post.id, "external_post_id": post.external_post_id, @@ -279,7 +288,10 @@ class PostFeedService: "description_plain": description_plain, "description_truncated": truncated, "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug}, - "source": {"id": source.id, "platform": source.platform}, + "source": ( + {"id": source.id, "platform": source.platform} + if source is not None else None + ), "thumbnails": thumbs_entry["thumbs"], "thumbnails_more": thumbs_entry["more"], "attachments": atts_map.get(post.id, []), diff --git a/backend/app/services/provenance_service.py b/backend/app/services/provenance_service.py index 091feef..9c2739b 100644 --- a/backend/app/services/provenance_service.py +++ b/backend/app/services/provenance_service.py @@ -70,11 +70,15 @@ class ProvenanceService: rec = await self.session.get(ImageRecord, image_id) if rec is None: return None + # Artist via Post.artist_id (alembic 0030); Source via LEFT JOIN + # since both Post.source_id and ImageProvenance.source_id can be + # NULL for filesystem-imported content. Frontend renders source= + # null as "filesystem import." stmt = ( select(ImageProvenance, Post, Source, Artist) .join(Post, Post.id == ImageProvenance.post_id) - .join(Source, Source.id == ImageProvenance.source_id) - .join(Artist, Artist.id == Source.artist_id) + .join(Artist, Artist.id == Post.artist_id) + .outerjoin(Source, Source.id == ImageProvenance.source_id) .where(ImageProvenance.image_record_id == image_id) .order_by(ImageProvenance.captured_at.asc(), ImageProvenance.id.asc()) @@ -90,7 +94,7 @@ class ProvenanceService: "captured_at": ip.captured_at.isoformat() if ip.captured_at else None, "post": _post_dict(post), - "source": _source_dict(src), + "source": _source_dict(src) if src is not None else None, "artist": _artist_dict(art), } for ip, post, src, art in rows @@ -99,10 +103,12 @@ class ProvenanceService: } async def for_post(self, post_id: int) -> dict | None: + # Same LEFT JOIN to Source — get_post must succeed for a + # NULL-source post. stmt = ( select(Post, Source, Artist) - .join(Source, Source.id == Post.source_id) - .join(Artist, Artist.id == Source.artist_id) + .join(Artist, Artist.id == Post.artist_id) + .outerjoin(Source, Source.id == Post.source_id) .where(Post.id == post_id) ) row = (await self.session.execute(stmt)).first() @@ -111,7 +117,7 @@ class ProvenanceService: post, src, art = row return { "post": _post_dict(post), - "source": _source_dict(src), + "source": _source_dict(src) if src is not None else None, "artist": _artist_dict(art), "attachments": await self._attachments_for_posts([post.id]), } diff --git a/backend/app/services/refetch_service.py b/backend/app/services/refetch_service.py index 41832bc..f2e941e 100644 --- a/backend/app/services/refetch_service.py +++ b/backend/app/services/refetch_service.py @@ -86,7 +86,7 @@ def attempt_refetch( if src is None: return {"status": "no_source"} - # Remove the bad copy so gallery-dl (skip_existing) re-fetches it on + # Remove the bad copy so gallery-dl's archive-skip re-fetches it on # the source re-check instead of skipping the still-present corrupt # file. try: diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 57a9571..144a994 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -62,6 +62,7 @@ class SourceRecord: check_interval_override: int | None consecutive_failures: int next_check_at: str | None + backfill_runs_remaining: int def to_dict(self) -> dict: return { @@ -78,6 +79,7 @@ class SourceRecord: "check_interval_override": self.check_interval_override, "consecutive_failures": self.consecutive_failures, "next_check_at": self.next_check_at, + "backfill_runs_remaining": self.backfill_runs_remaining, } @@ -85,6 +87,11 @@ class SourceRecord: _EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"} +# Plan #544 follow-up: newly created enabled sources pre-arm backfill so +# their first N polls walk gallery-dl's full post history with the longer +# timeout (matches the manual "Deep scan" button's default). +_NEW_SOURCE_BACKFILL_RUNS = 3 + class SourceService: def __init__(self, session: AsyncSession): @@ -140,6 +147,7 @@ class SourceService: check_interval_override=source.check_interval_override, consecutive_failures=source.consecutive_failures or 0, next_check_at=nxt.isoformat() if nxt else None, + backfill_runs_remaining=source.backfill_runs_remaining or 0, ) async def _row_to_record(self, source: Source) -> SourceRecord: @@ -157,7 +165,7 @@ class SourceService: if artist_id is not None: stmt = stmt.where(Source.artist_id == artist_id) if not include_synthetic: - # Filesystem-import sidecar anchors (importer._source_for_sidecar) + # Pre-alembic-0030 sidecar synthetic anchors # have url='sidecar::' and exist only to give # imported Posts a NOT-NULL Source FK. They aren't pollable # feeds; the Subscriptions UI used to render them as phantom @@ -201,10 +209,21 @@ class SourceService: select(func.count(Source.id)).where(Source.artist_id == artist_id) )).scalar_one() + # Plan #544 follow-up: a freshly added subscription has no archive + # yet, so the first few polls would walk the full post history in + # tick mode and trip exit:20 after ~20 contiguous archive hits — + # except there are none yet, so tick mode would walk forever and + # blow the wall-clock cap. Pre-arm backfill so the initial syncs + # use the longer timeout + skip:True walk. Tick mode resumes once + # the budget is spent or the queue drains. + # Disabled sources (incl. sidecar synthetics, url='sidecar:...') + # are never polled, so leave their counter at 0. + backfill_runs = _NEW_SOURCE_BACKFILL_RUNS if enabled else 0 source = Source( artist_id=artist_id, platform=platform, url=url, enabled=enabled, config_overrides=config_overrides, check_interval_override=check_interval_override, + backfill_runs_remaining=backfill_runs, ) self.session.add(source) try: @@ -264,6 +283,25 @@ class SourceService: await self.session.commit() return await self._row_to_record(source) + async def set_backfill_runs( + self, source_id: int, runs: int, + ) -> SourceRecord: + """Plan #544: arm a source for backfill mode. The next `runs` + download runs will use gallery-dl's full-walk config (skip: True + + 30-min timeout) instead of the catch-up default. Runs must be + 1..10 — bigger is rejected to keep the operator from accidentally + setting a runaway budget.""" + if not isinstance(runs, int) or runs < 1 or runs > 10: + raise ValueError("runs must be an integer in [1, 10]") + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + source.backfill_runs_remaining = runs + await self.session.commit() + return await self._row_to_record(source) + async def delete(self, source_id: int) -> None: source = (await self.session.execute( select(Source).where(Source.id == source_id) diff --git a/frontend/src/components/modal/ProvenancePanel.vue b/frontend/src/components/modal/ProvenancePanel.vue index 5c36fd4..51d16f2 100644 --- a/frontend/src/components/modal/ProvenancePanel.vue +++ b/frontend/src/components/modal/ProvenancePanel.vue @@ -19,7 +19,12 @@ v-for="e in state.entries" :key="e.provenance_id" class="fc-prov__card" >
- {{ e.source.platform }} + + + {{ e.source?.platform ?? 'filesystem import' }} + {{ postDate(e) }}
@@ -42,6 +48,16 @@ mdi-play Check now + + mdi-magnify-scan + + Deep scan — walk full history for next few runs + +
@@ -431,6 +432,36 @@ async function onCheck(source) { } } +// Plan #544: arm a source for backfill mode (gallery-dl walks the full +// post history) for the next N download runs. Default 3 — enough budget +// to finish a deep creator without re-prompting the operator across +// timeout boundaries. The chip on the row reflects the remaining count. +async function onBackfill(source) { + const raw = globalThis.window?.prompt( + `Deep scan "${source.artist_name} (${source.platform})" — walk full history for the next how many download runs? (1–10, default 3)`, + '3', + ) + if (raw == null) return + const runs = parseInt(raw, 10) + if (!Number.isFinite(runs) || runs < 1 || runs > 10) { + toast({ text: 'Deep scan: runs must be 1–10', type: 'error' }) + return + } + try { + await store.setBackfill(source.id, runs, source.artist_id) + toast({ + text: `Deep scan armed for ${runs} run${runs === 1 ? '' : 's'}`, + type: 'success', + }) + await store.loadAll() + } catch (e) { + toast({ + text: `Deep scan failed: ${e?.detail || e?.message || e}`, + type: 'error', + }) + } +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index a04c5a0..2e9406e 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -85,6 +85,15 @@ export const useSourcesStore = defineStore('sources', () => { } } + // Plan #544: arm a source for backfill mode. The next `runs` download + // runs (default 3) walk gallery-dl's full post history instead of + // exiting early at the first contiguous archived block. + async function setBackfill(id, runs = 3, artistIdHint = null) { + const body = await api.post(`/api/sources/${id}/backfill`, { body: { runs } }) + _invalidate(artistIdHint ?? body.artist_id) + return body + } + function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] const arr = byArtist.value.get(null) ?? [] @@ -111,6 +120,7 @@ export const useSourcesStore = defineStore('sources', () => { loadAll, loadForArtist, create, update, remove, checkNow, + setBackfill, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/test_api_artist.py b/tests/test_api_artist.py index f2b6587..9031df5 100644 --- a/tests/test_api_artist.py +++ b/tests/test_api_artist.py @@ -37,8 +37,8 @@ async def test_artist_overview_post_count(client, db): ) db.add(s) await db.flush() - db.add(Post(source_id=s.id, external_post_id="p1")) - db.add(Post(source_id=s.id, external_post_id="p2")) + db.add(Post(source_id=s.id, artist_id=a.id, external_post_id="p1")) + db.add(Post(source_id=s.id, artist_id=a.id, external_post_id="p2")) await db.flush() await db.commit() resp = await client.get("/api/artist/lyra") diff --git a/tests/test_api_posts.py b/tests/test_api_posts.py index ab0cad6..2791e9f 100644 --- a/tests/test_api_posts.py +++ b/tests/test_api_posts.py @@ -25,7 +25,7 @@ async def seeded_post(db): db.add(source) await db.flush() post = Post( - source_id=source.id, external_post_id="API1", + source_id=source.id, artist_id=artist.id, external_post_id="API1", post_title="Hello", post_url="https://p/alice-api/1", post_date=datetime.now(UTC), description="

hi

", @@ -123,7 +123,8 @@ async def post_timeline(db): posts = [] for i in range(5): p = Post( - source_id=source.id, external_post_id=f"TL{i}", + source_id=source.id, artist_id=artist.id, + external_post_id=f"TL{i}", post_title=f"post {i}", post_date=base + timedelta(days=i), ) db.add(p) @@ -189,7 +190,7 @@ async def test_detail_returns_uncapped_thumbnails(client, db): db.add(s) await db.flush() p = Post( - source_id=s.id, external_post_id="DETAIL10", + source_id=s.id, artist_id=a.id, external_post_id="DETAIL10", post_title="big post", description="

body

", ) db.add(p) diff --git a/tests/test_api_provenance.py b/tests/test_api_provenance.py index df0a2b7..47c49bf 100644 --- a/tests/test_api_provenance.py +++ b/tests/test_api_provenance.py @@ -26,7 +26,8 @@ async def _seed_full(db): url="https://patreon.test/alice") db.add(source) await db.flush() - post = Post(source_id=source.id, external_post_id="555", + post = Post(source_id=source.id, artist_id=artist.id, + external_post_id="555", post_url="https://patreon.test/p/555", post_title="Set 1", post_date=datetime(2023, 8, 1, tzinfo=UTC), description="

hi

", attachment_count=2) diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py index 28d1a11..c72c8b9 100644 --- a/tests/test_api_sources.py +++ b/tests/test_api_sources.py @@ -193,3 +193,61 @@ async def test_list_derives_next_check_at_when_last_checked_set( ) assert target["next_check_at"] is not None assert "T" in target["next_check_at"] # ISO 8601 + + +# --- Plan #544: POST /api/sources/{id}/backfill ---------------------------- + + +@pytest.mark.asyncio +async def test_backfill_endpoint_arms_source(client, artist, db): + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-backfill", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"runs": 5}) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["backfill_runs_remaining"] == 5 + + # GET reflects the new state. + one = await client.get(f"/api/sources/{sid}") + assert (await one.get_json())["backfill_runs_remaining"] == 5 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_defaults_to_three(client, artist, db): + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-backfill-default", enabled=True, + ) + db.add(src) + await db.commit() + resp = await client.post(f"/api/sources/{src.id}/backfill", json={}) + body = await resp.get_json() + assert body["backfill_runs_remaining"] == 3 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_rejects_out_of_range(client, artist, db): + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-backfill-bad", enabled=True, + ) + db.add(src) + await db.commit() + bad = await client.post(f"/api/sources/{src.id}/backfill", json={"runs": 0}) + assert bad.status_code == 400 + too_big = await client.post( + f"/api/sources/{src.id}/backfill", json={"runs": 99} + ) + assert too_big.status_code == 400 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_404_when_source_missing(client): + resp = await client.post("/api/sources/999999/backfill", json={"runs": 3}) + assert resp.status_code == 404 diff --git a/tests/test_artist_service.py b/tests/test_artist_service.py index 9858af9..2d29fff 100644 --- a/tests/test_artist_service.py +++ b/tests/test_artist_service.py @@ -25,7 +25,7 @@ async def _fixture(db): db.add(src) await db.flush() post = Post( - source_id=src.id, external_post_id="p1", + source_id=src.id, artist_id=artist.id, external_post_id="p1", post_date=datetime(2026, 3, 1, tzinfo=UTC), ) db.add(post) diff --git a/tests/test_download_service.py b/tests/test_download_service.py index 15ad337..ebced00 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -370,3 +370,179 @@ async def test_finalize_skipped_preserves_failures_clears_error(db): assert row.consecutive_failures == 4 assert row.last_error is None assert row.last_checked_at is not None + + +# --- Plan #544: backfill lifecycle + PARTIAL → status=ok ------------------- + + +@pytest.mark.asyncio +async def test_backfill_decrements_after_run( + db, db_sync, tmp_path, seed_artist_and_source, +): + """When backfill_runs_remaining > 0 going in, a non-clean / non-empty + run decrements by 1 — operator gets N runs to complete the deep scan + before tick mode resumes.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + source.backfill_runs_remaining = 3 + await db.commit() + + images_root = tmp_path / "images" + f1 = images_root / "alice" / "patreon" / "post" / "a.jpg" + _make_jpg(f1, split="h") + + fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + success=True, written_paths=[str(f1)], files_downloaded=1, + stdout=f"{f1}\n", + )) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=images_root, import_root=images_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + await svc.download_source(source.id) + + remaining = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == source.id) + )).scalar_one() + assert remaining == 2 + + +@pytest.mark.asyncio +async def test_backfill_auto_resets_on_clean_zero_files( + db, db_sync, tmp_path, seed_artist_and_source, +): + """A clean run (rc=0) that downloaded zero files means the backfill + queue drained — reset to 0 immediately instead of wasting the rest of + the N-run budget on no-op walks.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + source.backfill_runs_remaining = 3 + await db.commit() + + fake_result = _make_fake_dl_result( + success=True, written_paths=[], files_downloaded=0, + ) + fake_gdl = _fake_gdl_with_result(fake_result) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=tmp_path / "images", + import_root=tmp_path / "images", + thumbnailer=Thumbnailer(images_root=tmp_path / "images"), + settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + await svc.download_source(source.id) + + remaining = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == source.id) + )).scalar_one() + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_tick_mode_does_not_touch_backfill_counter( + db, db_sync, tmp_path, seed_artist_and_source, +): + """When backfill_runs_remaining is already 0, downloads don't go + negative or otherwise mutate the counter.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + assert source.backfill_runs_remaining == 0 + + fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + success=True, written_paths=[], files_downloaded=0, + )) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=tmp_path / "images", + import_root=tmp_path / "images", + thumbnailer=Thumbnailer(images_root=tmp_path / "images"), + settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + await svc.download_source(source.id) + + remaining = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == source.id) + )).scalar_one() + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_partial_error_type_maps_to_ok_status( + db, db_sync, tmp_path, seed_artist_and_source, +): + """A non-zero gallery-dl exit accompanied by PARTIAL error_type + (real files were downloaded before the run was cut short) is treated + as status=ok by the orchestrator — next tick continues.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.gallery_dl import ErrorType + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + + images_root = tmp_path / "images" + f1 = images_root / "alice" / "patreon" / "post" / "a.jpg" + _make_jpg(f1, split="h") + + fake_result = _make_fake_dl_result( + success=False, written_paths=[str(f1)], files_downloaded=1, + error_type=ErrorType.PARTIAL, + error_message="Downloaded 1 file; run did not complete in budget", + stdout=f"{f1}\n", + ) + fake_gdl = _fake_gdl_with_result(fake_result) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=images_root, import_root=images_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + event_id = await svc.download_source(source.id) + + ev = (await db.execute( + select(DownloadEvent).where(DownloadEvent.id == event_id) + )).scalar_one() + assert ev.status == "ok" + assert ev.error is None + # And the source's failure counter isn't bumped — PARTIAL isn't a failure. + src_after = (await db.execute( + select(Source.consecutive_failures).where(Source.id == source.id) + )).scalar_one() + assert src_after == 0 diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index af490d1..d3607a5 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -249,3 +249,80 @@ def test_truncate_log_caps_large_text(gdl): def test_truncate_log_passes_short_text(gdl): text = "small\n" assert gdl._truncate_log(text) == text + + +# --- Plan #544: tick/backfill skip emission + PARTIAL classifier ------------ + + +def test_build_config_emits_tick_skip_value(gdl): + """Tick mode emits gallery-dl's exit:20 to short-circuit catch-up + scans once 20 contiguous archived items are seen.""" + from backend.app.services.gallery_dl import TICK_SKIP_VALUE + cfg = gdl._build_config_for_source( + platform="patreon", + source_config=SourceConfig(), + artist_slug="alice", + skip_value=TICK_SKIP_VALUE, + ) + assert cfg["extractor"]["skip"] == "exit:20" + + +def test_build_config_emits_backfill_skip_value(gdl): + """Backfill mode keeps gallery-dl's default skip=True so it walks the + full post history.""" + from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE + cfg = gdl._build_config_for_source( + platform="patreon", + source_config=SourceConfig(), + artist_slug="alice", + skip_value=BACKFILL_SKIP_VALUE, + ) + assert cfg["extractor"]["skip"] is True + + +def test_categorize_partial_when_files_downloaded_then_nonzero_exit(gdl): + """Non-zero exit + ≥1 file downloaded + no source-level error → + PARTIAL. Downstream maps to status=ok because the next tick continues.""" + # stdout has a single downloaded file (line starting with `/`); no + # error-class indicators in stderr; non-zero return code. + etype, msg = gdl._categorize_error( + return_code=1, + stdout="/images/alice/patreon/post1/img.jpg\n", + stderr="", + ) + assert etype == ErrorType.PARTIAL + assert "Downloaded 1 file" in msg + + +def test_categorize_unknown_when_no_files_and_no_pattern(gdl): + """PARTIAL only kicks in when files were actually written. A non-zero + exit with no downloads and no recognized pattern still falls through + to UNKNOWN_ERROR — distinct from the partial-success case.""" + etype, _ = gdl._categorize_error( + return_code=2, + stdout="", + stderr="some unrecognized noise\n", + ) + assert etype == ErrorType.UNKNOWN_ERROR + + +def test_categorize_tier_limited_wins_over_partial(gdl): + """A run that hit tier-limited warnings AND downloaded some files + classifies as TIER_LIMITED (the more specific category), not PARTIAL.""" + stdout = "/images/alice/patreon/post1/img.jpg\n" + stderr = "[patreon][warning] Not allowed to view post 123\n" + etype, _ = gdl._categorize_error( + return_code=1, stdout=stdout, stderr=stderr, + ) + assert etype == ErrorType.TIER_LIMITED + + +def test_default_config_forwards_patreon_referer_to_ytdl(gdl): + """Operator-flagged 2026-06-01: Mux video playback restrictions reject + yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The + fix is a static `downloader.ytdl.raw-options.http_headers` block, so + it's enough to assert the keys land in the default config.""" + cfg = gdl._get_default_config() + headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"] + assert headers["Referer"] == "https://www.patreon.com/" + assert headers["Origin"] == "https://www.patreon.com" diff --git a/tests/test_gallery_provenance_filter.py b/tests/test_gallery_provenance_filter.py index e2fc95b..10b2ffa 100644 --- a/tests/test_gallery_provenance_filter.py +++ b/tests/test_gallery_provenance_filter.py @@ -35,7 +35,7 @@ async def _post(db, artist_name, slug, ext): url=f"https://patreon.test/{slug}") db.add(s) await db.flush() - p = Post(source_id=s.id, external_post_id=ext) + p = Post(source_id=s.id, artist_id=a.id, external_post_id=ext) db.add(p) await db.flush() return a, s, p diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py index 705747c..89d0c53 100644 --- a/tests/test_gallery_service.py +++ b/tests/test_gallery_service.py @@ -152,7 +152,8 @@ async def _seed_image_with_post( db.add(source) await db.flush() post = Post( - source_id=source.id, external_post_id=external_post_id, + source_id=source.id, artist_id=artist.id, + external_post_id=external_post_id, post_title="A Post", post_date=post_date, ) db.add(post) diff --git a/tests/test_importer_provenance_race.py b/tests/test_importer_provenance_race.py index 68ba96c..970a76e 100644 --- a/tests/test_importer_provenance_race.py +++ b/tests/test_importer_provenance_race.py @@ -29,7 +29,6 @@ from backend.app.models import ( ImageProvenance, ImageRecord, ImportSettings, - Source, ) from backend.app.services.importer import Importer from backend.app.services.thumbnailer import Thumbnailer @@ -139,9 +138,11 @@ def test_apply_sidecar_recovers_from_integrity_error( r = importer.import_one(m) assert r.status == "imported" rec = importer.session.get(ImageRecord, r.image_id) - src = importer.session.execute(select(Source)).scalar_one() + # alembic 0030 stopped creating synthetic Source rows for filesystem + # sidecars; the Post sits null-source and the provenance row points at + # it directly. The race-recovery path tested below operates on + # ImageProvenance regardless of Source presence. assert rec is not None - assert src is not None # Monkeypatch session.execute so the FIRST select inside _apply_sidecar's # existence-check returns a "no row" wrapper. Subsequent selects (e.g. diff --git a/tests/test_importer_upsert_helpers.py b/tests/test_importer_upsert_helpers.py index 7f8882e..04e2fc6 100644 --- a/tests/test_importer_upsert_helpers.py +++ b/tests/test_importer_upsert_helpers.py @@ -88,11 +88,36 @@ def test_find_or_create_post_idempotent(importer, artist_row, db_sync): ) p1 = importer._find_or_create_post( source_id=src.id, external_post_id="ext-001", + artist_id=artist_row.id, ) p2 = importer._find_or_create_post( source_id=src.id, external_post_id="ext-001", + artist_id=artist_row.id, ) assert p1.id == p2.id + assert p1.artist_id == artist_row.id + + +def test_find_or_create_post_idempotent_with_null_source( + importer, artist_row, db_sync, +): + """Post.source_id is nullable since alembic 0030 — filesystem-imported + posts with no live subscription have NULL source_id. The partial + unique index `uq_post_artist_external_id_null_source` on + (artist_id, external_post_id) WHERE source_id IS NULL guards the + dedup; the helper matches on the same (artist_id, external_post_id) + keys when source_id is None.""" + p1 = importer._find_or_create_post( + source_id=None, external_post_id="fs-001", + artist_id=artist_row.id, + ) + p2 = importer._find_or_create_post( + source_id=None, external_post_id="fs-001", + artist_id=artist_row.id, + ) + assert p1.id == p2.id + assert p1.source_id is None + assert p1.artist_id == artist_row.id def test_find_or_create_source_recovers_from_integrity_error( @@ -147,13 +172,13 @@ def test_find_or_create_source_recovers_from_integrity_error( assert recovered.id == pre_existing.id -def test_source_for_sidecar_reuses_existing_subscription( +def test_lookup_source_for_sidecar_returns_existing_subscription( importer, artist_row, db_sync, ): - """The filesystem-import sidecar resolver should attach to whatever - Source already exists for (artist, platform) — the canonical subscription - Source — regardless of its URL. Without this, every imported post - spawned its own Source row. + """The sidecar-import Source lookup returns the existing Source for + (artist, platform) when one exists — letting filesystem-imported + content attach to the artist's real subscription instead of being + orphaned. """ canonical = Source( artist_id=artist_row.id, platform="patreon", @@ -162,84 +187,30 @@ def test_source_for_sidecar_reuses_existing_subscription( db_sync.add(canonical) db_sync.flush() - resolved = importer._source_for_sidecar( + resolved = importer._lookup_source_for_sidecar( artist_id=artist_row.id, platform="patreon", - artist_slug=artist_row.slug, ) + assert resolved is not None assert resolved.id == canonical.id -def test_source_for_sidecar_creates_synthetic_anchor_when_none_exists( +def test_lookup_source_for_sidecar_returns_none_when_no_subscription( importer, artist_row, db_sync, ): - """No subscription Source for this (artist, platform) yet. The helper - creates one synthetic anchor (enabled=False, url='sidecar::') - so subsequent imports reuse it instead of spawning per-post Sources. + """Alembic 0030 made Post.source_id nullable; the importer no + longer creates synthetic `sidecar::` anchor rows + when no real subscription exists. The lookup returns None and the + caller carries that through as a NULL source_id on the new Post. """ - resolved = importer._source_for_sidecar( + resolved = importer._lookup_source_for_sidecar( artist_id=artist_row.id, platform="pixiv", - artist_slug=artist_row.slug, ) - assert resolved.url == f"sidecar:pixiv:{artist_row.slug}" - assert resolved.enabled is False - assert resolved.artist_id == artist_row.id - assert resolved.platform == "pixiv" + assert resolved is None - # Second call returns the same row (no new Source spawned). - again = importer._source_for_sidecar( - artist_id=artist_row.id, platform="pixiv", - artist_slug=artist_row.slug, - ) - assert again.id == resolved.id - - -def test_source_for_sidecar_distinct_platforms_distinct_anchors( - importer, artist_row, db_sync, -): - """One synthetic anchor per (artist, platform). Different platforms get - different anchors even when no campaign Source exists for either. - """ - p = importer._source_for_sidecar( - artist_id=artist_row.id, platform="patreon", - artist_slug=artist_row.slug, - ) - x = importer._source_for_sidecar( - artist_id=artist_row.id, platform="pixiv", - artist_slug=artist_row.slug, - ) - assert p.id != x.id - assert p.platform == "patreon" - assert x.platform == "pixiv" - - -def test_source_for_sidecar_prefers_real_over_synthetic_when_both_exist( - importer, artist_row, db_sync, -): - """When BOTH a synthetic anchor AND a real Source exist for the same - (artist, platform), the resolver must return the REAL one. This is the - fix for the 2026-05-31 phantom-subscription bug: alembic 0022 had - rewritten an older per-post Source row into a sidecar synthetic, and - the operator later added the real subscription. The old `ORDER BY - id ASC LIMIT 1` lookup picked the older synthetic (lower id), - silently attaching every gallery-dl download to the wrong Source. - """ - synthetic = Source( - artist_id=artist_row.id, platform="patreon", - url=f"sidecar:patreon:{artist_row.slug}", enabled=False, - ) - db_sync.add(synthetic) - db_sync.flush() - real = Source( - artist_id=artist_row.id, platform="patreon", - url="https://www.patreon.com/testartist", enabled=True, - ) - db_sync.add(real) - db_sync.flush() - assert synthetic.id < real.id # ordering precondition - - resolved = importer._source_for_sidecar( - artist_id=artist_row.id, platform="patreon", - artist_slug=artist_row.slug, - ) - assert resolved.id == real.id - assert resolved.url == "https://www.patreon.com/testartist" + # Verify no Source row was created as a side effect. + count = db_sync.execute( + select(Source).where( + Source.artist_id == artist_row.id, Source.platform == "pixiv", + ) + ).all() + assert count == [] diff --git a/tests/test_migration_0007.py b/tests/test_migration_0007.py index d452f2e..6f716f4 100644 --- a/tests/test_migration_0007.py +++ b/tests/test_migration_0007.py @@ -17,7 +17,7 @@ async def _post(db, **post_kwargs): db.add(src) await db.flush() post = Post( - source_id=src.id, external_post_id="p1", + source_id=src.id, artist_id=artist.id, external_post_id="p1", post_date=datetime(2026, 3, 1, tzinfo=UTC), **post_kwargs, ) diff --git a/tests/test_migration_0008.py b/tests/test_migration_0008.py index d3261b5..0d7552a 100644 --- a/tests/test_migration_0008.py +++ b/tests/test_migration_0008.py @@ -62,7 +62,7 @@ async def _run_backfill(db): async def test_backfill_primary_post(db): rec = await _img(db, 1) a, s = await _artist_source(db, "Alice", "alice") - post = Post(source_id=s.id, external_post_id="1") + post = Post(source_id=s.id, artist_id=a.id, external_post_id="1") db.add(post) await db.flush() rec.primary_post_id = post.id @@ -78,7 +78,7 @@ async def test_backfill_primary_post(db): async def test_backfill_provenance_fallback(db): rec = await _img(db, 1) a, s = await _artist_source(db, "Bob", "bob") - post = Post(source_id=s.id, external_post_id="2") + post = Post(source_id=s.id, artist_id=a.id, external_post_id="2") db.add(post) await db.flush() db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id, diff --git a/tests/test_phash_dedup.py b/tests/test_phash_dedup.py index 83b5725..da57ac2 100644 --- a/tests/test_phash_dedup.py +++ b/tests/test_phash_dedup.py @@ -15,7 +15,6 @@ from backend.app.models import ( ImageRecord, ImportSettings, Post, - Source, Tag, TagKind, ) @@ -197,10 +196,11 @@ def test_supersede_applies_new_file_sidecar(importer, import_layout): assert post.post_title == "Set 1" assert "big version" in (post.description or "") - source = importer.session.execute( - select(Source).where(Source.id == post.source_id) - ).scalar_one() - assert source.platform == "patreon" + # Filesystem sidecars no longer create a synthetic Source (alembic 0030). + # The Post sits null-source; the platform context is captured in the + # sidecar JSON / raw_metadata, not in a phantom Source row. + assert post.source_id is None + assert post.artist_id is not None prov_count = importer.session.execute( select(func.count(ImageProvenance.id)) diff --git a/tests/test_post_feed_service.py b/tests/test_post_feed_service.py index 651cbca..4924ab2 100644 --- a/tests/test_post_feed_service.py +++ b/tests/test_post_feed_service.py @@ -7,6 +7,7 @@ boundary, and the thumbnails/attachments composition. from datetime import UTC, datetime, timedelta import pytest +from sqlalchemy import select from backend.app.models import ( Artist, @@ -59,8 +60,13 @@ async def _seed_post( db, source_id: int, *, external_id: str, post_date=None, downloaded_at=None, title=None, description=None, ): + # Post.artist_id is NOT NULL since alembic 0030; look it up from + # the source so callsites don't have to thread the artist through. + artist_id = (await db.execute( + select(Source.artist_id).where(Source.id == source_id) + )).scalar_one() p = Post( - source_id=source_id, external_post_id=external_id, + source_id=source_id, artist_id=artist_id, external_post_id=external_id, post_title=title, post_date=post_date, description=description, ) diff --git a/tests/test_provenance_service.py b/tests/test_provenance_service.py index 7cc2aed..bd61081 100644 --- a/tests/test_provenance_service.py +++ b/tests/test_provenance_service.py @@ -37,7 +37,7 @@ async def _seed_post(db, *, artist_name, slug, platform, ext_id, db.add(source) await db.flush() post = Post( - source_id=source.id, external_post_id=ext_id, + source_id=source.id, artist_id=artist.id, external_post_id=ext_id, post_url=f"https://{platform}.test/p/{ext_id}", post_title=title, post_date=datetime(2023, 8, 1, tzinfo=UTC), description=desc, attachment_count=count, diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py index f9ebabc..1984d99 100644 --- a/tests/test_sidecar_import.py +++ b/tests/test_sidecar_import.py @@ -75,9 +75,15 @@ def test_sidecar_creates_provenance(importer, import_layout): r = importer.import_one(m) assert r.status == "imported" rec = importer.session.get(ImageRecord, r.image_id) - src = importer.session.execute(select(Source)).scalar_one() post = importer.session.execute(select(Post)).scalar_one() - assert src.platform == "patreon" + # Filesystem-imported sidecar posts no longer create a synthetic Source + # (alembic 0030 / nullable post.source_id refactor). The Post is linked + # to the artist via Post.artist_id; Post.source_id stays NULL until a + # real subscription for the (artist, platform) gets added. + assert post.source_id is None + assert importer.session.execute( + select(func.count()).select_from(Source) + ).scalar_one() == 0 assert post.external_post_id == "555" assert post.post_url == "https://patreon.com/posts/555" assert post.post_title == "Set 1" @@ -106,9 +112,11 @@ def test_reimport_same_post_idempotent(importer, import_layout): _sidecar(m2, payload) r2 = importer.import_one(m2) assert r2.status == "imported" + # No synthetic Source after alembic 0030; both imports still resolve to + # a single null-source Post (deduped by uq_post_artist_external_id_null_source). assert importer.session.execute( select(func.count()).select_from(Source) - ).scalar_one() == 1 + ).scalar_one() == 0 assert importer.session.execute( select(func.count()).select_from(Post) ).scalar_one() == 1 @@ -164,5 +172,11 @@ def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout): a = importer.session.execute( select(Artist).where(Artist.slug == "yuki") ).scalar_one() - src = importer.session.execute(select(Source)).scalar_one() - assert src.artist_id == a.id + # No synthetic Source after alembic 0030; the artist linkage lives on + # Post.artist_id (NOT NULL FK). + post = importer.session.execute(select(Post)).scalar_one() + assert post.artist_id == a.id + assert post.source_id is None + assert importer.session.execute( + select(func.count()).select_from(Source) + ).scalar_one() == 0 diff --git a/tests/test_source_service.py b/tests/test_source_service.py index f6df9f3..2806421 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -138,7 +138,7 @@ async def test_update_changes_fields(db): @pytest.mark.asyncio async def test_list_hides_sidecar_synthetic_anchors(db): """Filesystem-import synthetic Sources (url='sidecar::', - enabled=False — see importer._source_for_sidecar) used to leak into the + enabled=False — historical pre-alembic-0030 artifact) used to leak into the Subscriptions UI as phantom subscriptions because list() didn't filter them. They aren't pollable feeds; hide by default.""" artist = await _artist(db, "Alice") @@ -169,3 +169,75 @@ async def test_list_hides_sidecar_synthetic_anchors(db): assert {s.url for s in everything} >= { "https://patreon.com/alice", "sidecar:patreon:alice", } + + +# --- Plan #544: backfill counter ------------------------------------------- + + +@pytest.mark.asyncio +async def test_set_backfill_runs_arms_source(db): + """The service method overrides backfill_runs_remaining (regardless of + the auto-arm-on-create starting value) and returns the updated record + so the API can echo it back.""" + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + updated = await svc.set_backfill_runs(rec.id, 5) + assert updated.backfill_runs_remaining == 5 + + db_value = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == rec.id) + )).scalar_one() + assert db_value == 5 + + +@pytest.mark.asyncio +async def test_set_backfill_runs_rejects_out_of_range(db): + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + with pytest.raises(ValueError): + await svc.set_backfill_runs(rec.id, 0) + with pytest.raises(ValueError): + await svc.set_backfill_runs(rec.id, 11) + + +@pytest.mark.asyncio +async def test_set_backfill_runs_raises_when_source_missing(db): + svc = SourceService(db) + with pytest.raises(LookupError): + await svc.set_backfill_runs(99999, 3) + + +@pytest.mark.asyncio +async def test_new_enabled_source_starts_in_backfill_mode(db): + """Plan #544 follow-up: freshly added enabled sources have no archive + yet, so the first few polls would blow the wall-clock cap in tick + mode. Pre-arm backfill so the initial walks use the longer timeout.""" + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-new", + ) + assert rec.backfill_runs_remaining == 3 + + +@pytest.mark.asyncio +async def test_new_disabled_source_skips_backfill(db): + """Disabled sources (incl. sidecar synthetics that arrive disabled) are + never polled, so don't burn a backfill budget on them.""" + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-disabled", + enabled=False, + ) + assert rec.backfill_runs_remaining == 0