refactor(subscriptions): remove the dead backfill dry-run preview (#1281)
PreviewDialog.vue was orphaned — nothing mounted it — so the dry-run preview was unreachable. Rather than wire it up, remove the whole chain: its unique value (a capped 3-page count of what a backfill would grab) is low, and its adjacent needs are already covered — auth validation by verify_source_credential, and actually fetching recent posts by "Check now". Operator decision 2026-07-06. Removed: - frontend: PreviewDialog.vue + sources store previewSource() - backend: POST /api/sources/<id>/preview route, download_backends.preview_source, IngestCore.preview() + its now-unused NativeIngestError import - tests: the 3 ingester preview tests Nothing else referenced the chain (verified). Shared campaign-resolution and ledger helpers stay — they're used by run()/verify_source_credential. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -230,52 +230,6 @@ async def set_backfill(source_id: int):
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/preview", methods=["POST"])
|
||||
async def preview_source_endpoint(source_id: int):
|
||||
"""Plan #708 B4: dry-run — count what a backfill WOULD download for a native
|
||||
platform (Patreon today), without downloading. Walks the first few feed pages
|
||||
and counts media not already in the seen/dead ledgers. Returns
|
||||
{total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason
|
||||
(unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no
|
||||
cheap dry-run — their verify is a slow --simulate)."""
|
||||
from pathlib import Path
|
||||
|
||||
from ..services.credential_service import CredentialService
|
||||
from ..services.download_backends import preview_source, uses_native_ingester
|
||||
from ..tasks._sync_engine import sync_session_factory
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
rec = await SourceService(session).get(source_id)
|
||||
if rec is None:
|
||||
return _bad("not_found", status=404)
|
||||
if not uses_native_ingester(rec.platform):
|
||||
return _bad(
|
||||
"unsupported",
|
||||
detail="Preview is only available for native-ingester platforms.",
|
||||
status=400,
|
||||
)
|
||||
cred = CredentialService(session, _get_crypto())
|
||||
cookies_path = await cred.get_cookies_path(rec.platform)
|
||||
auth_token = await cred.get_token(rec.platform)
|
||||
|
||||
# The walk + ledger reads are sync (run off the request loop); the process
|
||||
# sync engine is the same one the download task uses.
|
||||
result = await preview_source(
|
||||
platform=rec.platform,
|
||||
url=rec.url,
|
||||
source_id=source_id,
|
||||
config_overrides=rec.config_overrides or {},
|
||||
cookies_path=str(cookies_path) if cookies_path else None,
|
||||
images_root=Path("/images"),
|
||||
sync_session_factory=sync_session_factory(),
|
||||
auth_token=auth_token,
|
||||
)
|
||||
if "error" in result:
|
||||
return _bad("preview_failed", detail=result["error"], status=409)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
||||
async def check_source(source_id: int):
|
||||
"""FC-3c: enqueue a download for this source.
|
||||
|
||||
@@ -24,7 +24,6 @@ import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .native_ingest_common import NativeIngestError
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||
from .pixiv_client import user_id_from_url
|
||||
@@ -204,49 +203,6 @@ async def _run_native_ingester(
|
||||
return dl_result, resolved_campaign_id
|
||||
|
||||
|
||||
async def preview_source(
|
||||
*,
|
||||
platform: str,
|
||||
url: str,
|
||||
source_id: int,
|
||||
config_overrides: dict | None,
|
||||
cookies_path: str | None,
|
||||
images_root: Path,
|
||||
sync_session_factory,
|
||||
auth_token: str | None = None,
|
||||
page_limit: int = 3,
|
||||
) -> dict:
|
||||
"""Dry-run preview for a native platform (plan #708 B4): resolve the campaign
|
||||
id, then walk a few pages counting media not already seen/dead — no download.
|
||||
|
||||
Returns the preview dict (total_new / posts_scanned / pages_scanned /
|
||||
has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
|
||||
Native-only — the caller gates on `uses_native_ingester`.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
campaign_id, _ = await _resolve_native_campaign_id(
|
||||
platform, url, cookies_path, config_overrides or {}
|
||||
)
|
||||
if not campaign_id:
|
||||
return {"error": _campaign_resolution_error(platform, url)}
|
||||
ingester = _native_ingester_cls(platform)(
|
||||
images_root=images_root,
|
||||
cookies_path=cookies_path,
|
||||
session_factory=sync_session_factory,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
result = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
|
||||
)
|
||||
except NativeIngestError as exc:
|
||||
return {"error": f"Couldn't preview: {exc}"}
|
||||
return result
|
||||
|
||||
|
||||
async def verify_source_credential(
|
||||
*,
|
||||
platform: str,
|
||||
|
||||
@@ -577,73 +577,6 @@ class Ingester:
|
||||
error_type=None, error_message=None,
|
||||
)
|
||||
|
||||
# -- preview (dry-run) -------------------------------------------------
|
||||
|
||||
def preview(
|
||||
self,
|
||||
source_id: int,
|
||||
campaign_id: str,
|
||||
*,
|
||||
page_limit: int = 3,
|
||||
sample_size: int = 10,
|
||||
) -> dict:
|
||||
"""Dry-run (plan #708 B4): walk up to `page_limit` pages and count media
|
||||
NOT already in the seen/dead ledgers, WITHOUT downloading anything.
|
||||
|
||||
Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets
|
||||
an operator gauge "is this source worth a backfill?" cheaply. Returns:
|
||||
{total_new, posts_scanned, pages_scanned, has_more,
|
||||
sample: [{title, date, new}, ...]} # sample = posts with new media
|
||||
A client-level failure (auth/drift) propagates to the caller.
|
||||
"""
|
||||
total_new = 0
|
||||
posts_scanned = 0
|
||||
pages_scanned = 0
|
||||
has_more = False
|
||||
sample: list[dict] = []
|
||||
unset = object()
|
||||
last_page: object = unset
|
||||
# #874: same gated-post gate as run() — the preview must not count
|
||||
# blurred locked-preview media as "new", or it would overstate a gated
|
||||
# source's backlog (preview/apply parity, rule 93).
|
||||
post_is_gated = getattr(self.client, "post_is_gated", None)
|
||||
for post, included, page_cursor in self.client.iter_posts(
|
||||
campaign_id, cursor=None
|
||||
):
|
||||
if page_cursor != last_page:
|
||||
last_page = page_cursor
|
||||
pages_scanned += 1
|
||||
if pages_scanned > page_limit:
|
||||
has_more = True
|
||||
pages_scanned = page_limit
|
||||
break
|
||||
posts_scanned += 1
|
||||
if post_is_gated and post_is_gated(post):
|
||||
continue
|
||||
media = self.client.extract_media(post, included)
|
||||
if not media:
|
||||
continue
|
||||
keys = [self._ledger_key(m) for m in media]
|
||||
skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys)
|
||||
new_count = sum(1 for m in media if self._ledger_key(m) not in skip)
|
||||
total_new += new_count
|
||||
if new_count > 0 and len(sample) < sample_size:
|
||||
meta = self.client.post_meta(post)
|
||||
sample.append(
|
||||
{
|
||||
"title": meta.get("title") or "(untitled)",
|
||||
"date": meta.get("date"),
|
||||
"new": new_count,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"total_new": total_new,
|
||||
"posts_scanned": posts_scanned,
|
||||
"pages_scanned": pages_scanned,
|
||||
"has_more": has_more,
|
||||
"sample": sample,
|
||||
}
|
||||
|
||||
# -- failure mapping (adapter overrides) -------------------------------
|
||||
|
||||
def _failure_result(self, exc: Exception, _result) -> DownloadResult:
|
||||
|
||||
Reference in New Issue
Block a user