Pixiv native ingester (#129) + SubscribeStar .art fix, disable-clears-failure, system-tag suggestion floor #191

Merged
bvandeusen merged 9 commits from dev into main 2026-07-03 16:36:56 -04:00
4 changed files with 104 additions and 2 deletions
Showing only changes of commit c9696a2faf - Show all commits
+9
View File
@@ -298,6 +298,15 @@ class SourceService:
for key, value in fields.items():
setattr(source, key, value)
# Disabling a source clears its failure state (operator: disable the subs
# you're not paying for without them lingering as "failing"). Re-enabling
# then starts clean; the next real run re-derives health. Only on the
# explicit disable — an unrelated edit to an already-disabled source
# leaves its (already-clear) state alone.
if fields.get("enabled") is False:
source.last_error = None
source.error_type = None
source.consecutive_failures = 0
try:
await self.session.flush()
except IntegrityError as exc:
+24 -2
View File
@@ -226,13 +226,35 @@ def _attachment_item(
)
def _normalize_ss_host(netloc: str) -> str:
"""Rewrite the `subscribestar.art` host to `subscribestar.adult`.
The age wall on the `.art` domain does not clear with the
`18_plus_agreement_generic` cookie (unlike `.com`/`.adult`): a `.art`
creator page keeps 302'ing to `/age_confirmation_warning` even with the
cookie set (Elasid, event #54116). The same creator is reachable on
`.adult`, where the cookie works — so `.art` behaves as an alias that
doesn't honor the age gate. Normalize it to `.adult` at request time (the
stored Source.url is left untouched). `.com`/`.adult` pass through.
"""
host = netloc.lower()
if host == "subscribestar.art" or host.endswith(".subscribestar.art"):
rewritten = netloc[: -len("art")] + "adult"
log.info(
"SubscribeStar: rewrote age-gated .art host %r%r", netloc, rewritten
)
return rewritten
return netloc
def _split_creator_url(campaign_id: str) -> tuple[str, str]:
"""`campaign_id` is the creator URL → (base, slug).
base = scheme://host (preserving .com vs .adult); slug = first path segment.
base = scheme://host (preserving .com vs .adult; .art → .adult, see
_normalize_ss_host); slug = first path segment.
"""
parts = urlsplit(campaign_id)
base = f"{parts.scheme or 'https'}://{parts.netloc}"
base = f"{parts.scheme or 'https'}://{_normalize_ss_host(parts.netloc)}"
slug = parts.path.strip("/").split("/")[0] if parts.path else ""
return base, slug
+51
View File
@@ -135,6 +135,57 @@ async def test_update_changes_fields(db):
assert updated.config_overrides == {"videos": False}
@pytest.mark.asyncio
async def test_disable_clears_failure_state(db):
"""Disabling a source wipes its failure state so it stops showing as
'failing' (operator: disable subs you're not paying for)."""
artist = await _artist(db)
svc = SourceService(db)
rec = await svc.create(
artist_id=artist.id, platform="patreon", url="https://patreon.com/a",
)
source = (await db.execute(
select(Source).where(Source.id == rec.id)
)).scalar_one()
source.last_error = "auth failed"
source.error_type = "auth_error"
source.consecutive_failures = 5
await db.commit()
updated = await svc.update(rec.id, enabled=False)
assert updated.enabled is False
assert updated.last_error is None
assert updated.consecutive_failures == 0
refetched = (await db.execute(
select(Source).where(Source.id == rec.id)
)).scalar_one()
assert refetched.error_type is None
@pytest.mark.asyncio
async def test_update_while_enabled_keeps_failure_state(db):
"""A non-disable edit must NOT wipe failure state — only the explicit
disable clears it (else a config tweak would hide a real failure)."""
artist = await _artist(db)
svc = SourceService(db)
rec = await svc.create(
artist_id=artist.id, platform="patreon", url="https://patreon.com/a",
)
source = (await db.execute(
select(Source).where(Source.id == rec.id)
)).scalar_one()
source.last_error = "auth failed"
source.consecutive_failures = 3
await db.commit()
await svc.update(rec.id, config_overrides={"videos": False})
refetched = (await db.execute(
select(Source).where(Source.id == rec.id)
)).scalar_one()
assert refetched.last_error == "auth failed"
assert refetched.consecutive_failures == 3
@pytest.mark.asyncio
async def test_list_hides_sidecar_synthetic_anchors(db):
"""Filesystem-import synthetic Sources (url='sidecar:<platform>:<slug>',
+20
View File
@@ -22,6 +22,7 @@ from backend.app.services.subscribestar_client import (
SubscribeStarClient,
SubscribeStarDriftError,
_parse_ss_datetime,
_split_creator_url,
)
from backend.app.services.subscribestar_downloader import SubscribeStarDownloader
from backend.app.services.subscribestar_ingester import (
@@ -82,6 +83,25 @@ def test_parse_ss_datetime_variants():
assert _parse_ss_datetime("nonsense") is None
def test_split_creator_url_rewrites_art_to_adult():
# The .art age wall doesn't clear with the 18+ cookie; the same creator is
# reachable on .adult (Elasid, event #54116). Requests must route to .adult.
base, slug = _split_creator_url("https://subscribestar.art/elasid")
assert base == "https://subscribestar.adult"
assert slug == "elasid"
# www. prefix + trailing path still normalizes the host only.
base, slug = _split_creator_url("https://www.subscribestar.art/elasid/posts")
assert base == "https://www.subscribestar.adult"
assert slug == "elasid"
def test_split_creator_url_leaves_com_and_adult_untouched():
assert _split_creator_url("https://subscribestar.adult/sabu")[0] == \
"https://subscribestar.adult"
assert _split_creator_url("https://www.subscribestar.com/sabu")[0] == \
"https://www.subscribestar.com"
def test_date_parses_when_wrapped_in_permalink_anchor():
# Image posts wrap the date in an <a> permalink; a plain regex missed them
# → null dates (cheunart 2026-06-17). gallery-dl's method handles both.