Compare commits
24
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0533807669 | ||
|
|
d3245f0c22 | ||
|
|
279dff3fb6 | ||
|
|
e450145304 | ||
|
|
a6e8d4b52e | ||
|
|
37e66cddc4 | ||
|
|
f1860866de | ||
|
|
9cf6b2d363 | ||
|
|
b181d779fe | ||
|
|
0fbb19dc24 | ||
|
|
8326e5447a | ||
|
|
1fd594baaf | ||
|
|
ecac6c4bda | ||
|
|
6ef0fed41f | ||
|
|
9f7261b9c0 | ||
|
|
f05aaa707b | ||
|
|
4df98171ab | ||
|
|
8d75ade1d5 | ||
|
|
75c63e1511 | ||
|
|
98673d4dca | ||
|
|
89b48f8f35 | ||
|
|
4bff1d8558 | ||
|
|
d60e0b9494 | ||
|
|
e30f50e6fe |
@@ -0,0 +1,41 @@
|
||||
"""source.error_type: surface ErrorType taxonomy in FailingSourcesCard
|
||||
|
||||
Revision ID: 0032
|
||||
Revises: 0031
|
||||
Create Date: 2026-06-02
|
||||
|
||||
Audit 2026-06-02: the backend computes 13 ErrorType categories (auth_error,
|
||||
rate_limited, not_found, access_denied, validation_failed, etc.) and
|
||||
stamps each one on DownloadEvent.metadata, but the Source row only carried
|
||||
the free-text last_error. Operators couldn't bulk-triage failing sources
|
||||
("all auth_error → rotate cookies, all rate_limited → just wait") without
|
||||
opening Logs per row.
|
||||
|
||||
This column receives the last error_type from _update_source_health
|
||||
and gets cleared on a successful run. Nullable + indexed so the failing-
|
||||
sources rollup can filter/group cheaply.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0032"
|
||||
down_revision: Union[str, None] = "0031"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"source",
|
||||
sa.Column("error_type", sa.String(length=32), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_source_error_type", "source", ["error_type"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_source_error_type", table_name="source")
|
||||
op.drop_column("source", "error_type")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""suggestion_threshold default 0.50 → 0.70
|
||||
|
||||
Revision ID: 0033
|
||||
Revises: 0032
|
||||
Create Date: 2026-06-02
|
||||
|
||||
Operator-flagged 2026-06-02 — the 0.50 default (set on 2026-06-01) is
|
||||
too noisy in practice; raise to 0.70 for both suggestion categories.
|
||||
|
||||
Only conditionally updates singletons whose current value is still the
|
||||
2026-06-01 default (0.50). Operators who deliberately tuned their row
|
||||
to some other value (0.55, 0.65, 0.80, etc. via the Settings UI) keep
|
||||
their pick — the migration only catches the unchanged-default case.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0033"
|
||||
down_revision: Union[str, None] = "0032"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE ml_settings "
|
||||
"SET suggestion_threshold_character = 0.70 "
|
||||
"WHERE id = 1 AND suggestion_threshold_character = 0.50"
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE ml_settings "
|
||||
"SET suggestion_threshold_general = 0.70 "
|
||||
"WHERE id = 1 AND suggestion_threshold_general = 0.50"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE ml_settings "
|
||||
"SET suggestion_threshold_character = 0.50 "
|
||||
"WHERE id = 1 AND suggestion_threshold_character = 0.70"
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE ml_settings "
|
||||
"SET suggestion_threshold_general = 0.50 "
|
||||
"WHERE id = 1 AND suggestion_threshold_general = 0.70"
|
||||
)
|
||||
@@ -43,7 +43,6 @@ async def scroll():
|
||||
"height": i.height,
|
||||
"created_at": i.created_at.isoformat(),
|
||||
"posted_at": i.posted_at.isoformat() if i.posted_at else None,
|
||||
"effective_date": i.effective_date.isoformat(),
|
||||
"thumbnail_url": i.thumbnail_url,
|
||||
"artist": i.artist,
|
||||
}
|
||||
|
||||
@@ -35,10 +35,26 @@ async def trigger_scan():
|
||||
@import_admin_bp.route("/status", methods=["GET"])
|
||||
async def status():
|
||||
async with get_session() as session:
|
||||
# Active batch = running batch that still has outstanding work.
|
||||
# Plain "most recent running" picks freshly-created scans that
|
||||
# enqueued zero new files and hides the older batch that's
|
||||
# actually being processed. Mirrors the EXISTS predicate
|
||||
# /api/system/stats already uses (api/settings.py:145-160).
|
||||
# Audit 2026-06-02 — /api/import/status and /api/system/stats
|
||||
# used to disagree on the active-batch predicate; the UI banner
|
||||
# said "Scanning…" indefinitely while the stats card said idle.
|
||||
active = (
|
||||
await session.execute(
|
||||
select(ImportBatch)
|
||||
.where(ImportBatch.status == "running")
|
||||
.where(
|
||||
ImportBatch.status == "running",
|
||||
select(ImportTask.id)
|
||||
.where(
|
||||
ImportTask.batch_id == ImportBatch.id,
|
||||
ImportTask.status.in_(["pending", "queued", "processing"]),
|
||||
)
|
||||
.exists(),
|
||||
)
|
||||
.order_by(ImportBatch.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
@@ -245,6 +245,12 @@ async def merge_tag(source_id: int):
|
||||
from ..tasks.ml import apply_allowlist_tags
|
||||
|
||||
apply_allowlist_tags.delay(tag_id=result.target_id)
|
||||
# Tag merge invalidates the target's centroid (the merged-in source
|
||||
# tag's images now contribute to it). Daily list_drifted catches it
|
||||
# within 24h, but eager recompute closes the suggestion-quality dip
|
||||
# in the meantime. Audit 2026-06-02.
|
||||
from ..tasks.ml import recompute_centroid
|
||||
recompute_centroid.delay(result.target_id)
|
||||
return jsonify(
|
||||
{
|
||||
"target": {
|
||||
|
||||
@@ -105,6 +105,41 @@ def make_celery() -> Celery:
|
||||
"task": "backend.app.tasks.backup.prune_backups",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
# Audit 2026-06-02 — three new per-entity recovery sweeps.
|
||||
# Each runs every 5 min like the other recover_stalled_*
|
||||
# sweeps; each is a no-op when nothing is stuck.
|
||||
"recover-stalled-backup-runs": {
|
||||
"task": "backend.app.tasks.maintenance.recover_stalled_backup_runs",
|
||||
"schedule": 300.0,
|
||||
},
|
||||
"recover-stalled-library-audit-runs": {
|
||||
"task": "backend.app.tasks.maintenance.recover_stalled_library_audit_runs",
|
||||
"schedule": 300.0,
|
||||
},
|
||||
"recover-stalled-import-batches": {
|
||||
"task": "backend.app.tasks.maintenance.recover_stalled_import_batches",
|
||||
"schedule": 300.0,
|
||||
},
|
||||
# Audit 2026-06-02 — daily retention for two entities
|
||||
# whose terminal rows otherwise accumulate forever.
|
||||
"prune-library-audit-runs": {
|
||||
"task": "backend.app.tasks.maintenance.prune_library_audit_runs",
|
||||
"schedule": 86400.0,
|
||||
},
|
||||
"prune-import-batches": {
|
||||
"task": "backend.app.tasks.maintenance.prune_import_batches",
|
||||
"schedule": 86400.0,
|
||||
},
|
||||
# Audit 2026-06-02 — backfill_thumbnails's docstring claimed
|
||||
# "periodic Beat" but the entry was never registered, so the
|
||||
# library got no self-healing thumbnail repair; only the
|
||||
# manual admin-UI button fired it. Daily cadence is gentle
|
||||
# (the task is idempotent and only enqueues regen for rows
|
||||
# whose stored thumbnails are missing or corrupt).
|
||||
"backfill-thumbnails-daily": {
|
||||
"task": "backend.app.tasks.thumbnail.backfill_thumbnails",
|
||||
"schedule": 86400.0,
|
||||
},
|
||||
},
|
||||
timezone="UTC",
|
||||
)
|
||||
|
||||
@@ -54,7 +54,14 @@ _INT32_MIN = -2_147_483_648
|
||||
|
||||
def _queue_for(task) -> str:
|
||||
"""Reverse the task→queue routing from celery_app.task_routes.
|
||||
Keep in sync if task_routes is reordered."""
|
||||
Keep in sync if task_routes is reordered.
|
||||
|
||||
Audit 2026-06-02: backup/admin/library_audit prefixes were
|
||||
missing here even though task_routes sent all three to
|
||||
'maintenance'. The TaskRun.queue column then lied for those
|
||||
rows (claimed 'default') so per-queue dashboard filters and
|
||||
per-queue threshold overrides silently missed them.
|
||||
"""
|
||||
name = getattr(task, "name", "") or ""
|
||||
if name.startswith("backend.app.tasks.import_file."):
|
||||
return "import"
|
||||
@@ -66,7 +73,12 @@ def _queue_for(task) -> str:
|
||||
return "download"
|
||||
if name.startswith("backend.app.tasks.scan."):
|
||||
return "scan"
|
||||
if name.startswith("backend.app.tasks.maintenance."):
|
||||
if name.startswith((
|
||||
"backend.app.tasks.maintenance.",
|
||||
"backend.app.tasks.backup.",
|
||||
"backend.app.tasks.admin.",
|
||||
"backend.app.tasks.library_audit.",
|
||||
)):
|
||||
return "maintenance"
|
||||
return "default"
|
||||
|
||||
|
||||
@@ -16,13 +16,14 @@ class MLSettings(Base):
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
suggestion_threshold_character: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
Float, nullable=False, default=0.70
|
||||
)
|
||||
# Default lowered 0.95 → 0.50 on 2026-06-01 — operator-flagged that
|
||||
# 0.95 hid most general suggestions. Operator-tunable via Settings →
|
||||
# ML if too noisy.
|
||||
# Default raised 0.50 → 0.70 on 2026-06-02 — operator-flagged 0.50
|
||||
# surfaced too many low-confidence picks; 0.70 keeps the rail
|
||||
# signal-rich while still surfacing more than the original 0.95
|
||||
# which hid almost everything. Operator-tunable via Settings → ML.
|
||||
suggestion_threshold_general: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
Float, nullable=False, default=0.70
|
||||
)
|
||||
centroid_similarity_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.55
|
||||
|
||||
@@ -26,6 +26,11 @@ class Source(Base):
|
||||
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# alembic 0032: last ErrorType category (auth_error, rate_limited,
|
||||
# not_found, ...). Lets FailingSourcesCard surface the taxonomy as
|
||||
# a colored chip so operators can bulk-triage by error class. Set
|
||||
# by _update_source_health alongside last_error; cleared on 'ok'.
|
||||
error_type: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
|
||||
@@ -208,27 +208,32 @@ class ArtistService:
|
||||
)
|
||||
|
||||
async def find_or_create(self, name: str) -> tuple[Artist, bool]:
|
||||
"""Return (artist, created). Slug-keyed; idempotent under races."""
|
||||
"""Return (artist, created). Slug-keyed; idempotent under races.
|
||||
|
||||
Audit 2026-06-02: switched from session.rollback() to a
|
||||
begin_nested savepoint + IntegrityError recovery so a lost
|
||||
race doesn't unwind the calling request's surrounding work.
|
||||
Mirrors importer._get_or_create.
|
||||
"""
|
||||
cleaned = (name or "").strip()
|
||||
if not cleaned:
|
||||
raise ValueError("artist name must not be empty")
|
||||
slug = slugify(cleaned)
|
||||
|
||||
existing = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
select_existing = select(Artist).where(Artist.slug == slug)
|
||||
existing = (await self.session.execute(select_existing)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
|
||||
artist = Artist(name=cleaned, slug=slug)
|
||||
self.session.add(artist)
|
||||
sp = await self.session.begin_nested()
|
||||
try:
|
||||
artist = Artist(name=cleaned, slug=slug)
|
||||
self.session.add(artist)
|
||||
await self.session.flush()
|
||||
await sp.commit()
|
||||
except IntegrityError:
|
||||
await self.session.rollback()
|
||||
existing = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one()
|
||||
await sp.rollback()
|
||||
existing = (await self.session.execute(select_existing)).scalar_one()
|
||||
return existing, False
|
||||
await self.session.commit()
|
||||
return artist, True
|
||||
|
||||
@@ -11,7 +11,7 @@ the one-and-done GS/IR migration tooling.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -187,10 +187,14 @@ def unlink_image_files(
|
||||
out["thumbnail"] = True
|
||||
except OSError:
|
||||
out["thumbnail"] = False
|
||||
# Convention thumbs dir — try all extensions; missing OK.
|
||||
# Convention thumbs dir — try both extensions thumbnailer writes
|
||||
# (.jpg for opaque, .png for alpha). `.webp` used to be in this
|
||||
# tuple but the thumbnailer never writes it (operator-flagged in
|
||||
# the 2026-06-02 audit) — keep the tuple aligned with what
|
||||
# actually lands on disk.
|
||||
if image.sha256:
|
||||
bucket = image.sha256[:3]
|
||||
for ext in ("jpg", "png", "webp"):
|
||||
for ext in ("jpg", "png"):
|
||||
try:
|
||||
(images_root / "thumbs" / bucket / f"{image.sha256}.{ext}").unlink(
|
||||
missing_ok=True,
|
||||
@@ -517,6 +521,9 @@ class ConfirmTokenMismatch(Exception):
|
||||
_VALID_RULES = ("transparency", "single_color")
|
||||
|
||||
|
||||
_AUDIT_GUARD_THRESHOLD_MINUTES = 135 # matches LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES
|
||||
|
||||
|
||||
def start_audit_run(
|
||||
session: Session, *, rule: str, params: dict[str, Any],
|
||||
) -> int:
|
||||
@@ -524,11 +531,21 @@ def start_audit_run(
|
||||
scan_library_for_rule Celery task. Returns the new audit_id.
|
||||
|
||||
Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run
|
||||
has status='running'. Operator must cancel or wait."""
|
||||
has status='running' AND started recently. Audit 2026-06-02 made
|
||||
the guard age-aware: a SIGKILL'd run leaves a row in 'running'
|
||||
that the recovery sweep flips on its next pass (~5 min), but a
|
||||
fresh start_audit_run between the SIGKILL and the sweep would
|
||||
previously block forever. Past the threshold, treat the running
|
||||
row as stale and let the sweep clean it up — the new run still
|
||||
gets to start.
|
||||
"""
|
||||
if rule not in _VALID_RULES:
|
||||
raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}")
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=_AUDIT_GUARD_THRESHOLD_MINUTES)
|
||||
existing = session.execute(
|
||||
select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running")
|
||||
select(LibraryAuditRun.id)
|
||||
.where(LibraryAuditRun.status == "running")
|
||||
.where(LibraryAuditRun.started_at >= cutoff)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise AuditAlreadyRunning(existing)
|
||||
|
||||
@@ -1,40 +1,82 @@
|
||||
"""Fernet-based encryption for credential blobs.
|
||||
|
||||
The key is a single 32-byte value (urlsafe-base64-encoded; what
|
||||
Fernet.generate_key produces) stored at a fixed path inside the
|
||||
images/data root. Created on first boot if absent; mode 0600. No KDF
|
||||
needed — the file contents are already maximum-entropy random bytes.
|
||||
Fernet.generate_key produces) stored at /images/secrets/credential_key.b64
|
||||
(mode 0600, parent dir 0700). The 2026-06-02 audit caught a silent
|
||||
key-regeneration path: on a partial disaster restore where the DB was
|
||||
restored but the secrets dir was lost, the old `_load_or_create_key`
|
||||
would mint a fresh key with no log, producing a working-looking system
|
||||
where every authenticated download failed AUTH_ERROR until the operator
|
||||
re-uploaded every credential by hand. Now the constructor refuses to
|
||||
auto-generate unless either:
|
||||
|
||||
Operator backup procedure must include this file alongside the rest
|
||||
of /images/ — losing it makes existing encrypted_blob rows
|
||||
undecryptable (recovery = delete the rows and re-upload).
|
||||
* the caller explicitly passes `bootstrap_ok=True` (tests, scripts), or
|
||||
* the env var `CURATOR_BOOTSTRAP_NEW_KEY=1` is set (operator opt-in
|
||||
during first-time setup).
|
||||
|
||||
Otherwise it raises `MissingCredentialKey` so the app fails fast at
|
||||
startup and the operator can restore the key file from backup.
|
||||
|
||||
Operator backup procedure must include /images/secrets/ alongside the
|
||||
rest of /images/ — losing the key file makes existing encrypted_blob
|
||||
rows undecryptable (recovery = delete the rows and re-upload).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BOOTSTRAP_ENV_VAR = "CURATOR_BOOTSTRAP_NEW_KEY"
|
||||
|
||||
|
||||
class InvalidCredentialBlob(Exception):
|
||||
"""Raised when decryption fails (wrong key, tampered blob, …)."""
|
||||
|
||||
|
||||
class MissingCredentialKey(Exception):
|
||||
"""The Fernet key file is missing AND the caller hasn't opted in to
|
||||
generating a new one. Audit 2026-06-02: prevents silent key
|
||||
regeneration on partial DB-restored / secrets-lost deployments.
|
||||
Set CURATOR_BOOTSTRAP_NEW_KEY=1 for first-time setup, or restore the
|
||||
key file from backup."""
|
||||
|
||||
|
||||
class CredentialCrypto:
|
||||
"""Fernet encrypt/decrypt with an on-disk key file.
|
||||
|
||||
Instantiate with a path; the file is created on first access and
|
||||
reused thereafter. Tests pass a tmp_path; production calls with
|
||||
Instantiate with a path; the file is loaded if present, or created
|
||||
if absent AND the caller has opted in (bootstrap_ok=True or
|
||||
CURATOR_BOOTSTRAP_NEW_KEY=1 env var). Production sites:
|
||||
`IMAGES_ROOT / "secrets" / "credential_key.b64"`.
|
||||
"""
|
||||
|
||||
def __init__(self, key_path: Path):
|
||||
def __init__(self, key_path: Path, *, bootstrap_ok: bool | None = None):
|
||||
self._key_path = Path(key_path)
|
||||
self._fernet = Fernet(self._load_or_create_key())
|
||||
if bootstrap_ok is None:
|
||||
bootstrap_ok = os.environ.get(_BOOTSTRAP_ENV_VAR) == "1"
|
||||
self._fernet = Fernet(self._load_or_create_key(bootstrap_ok))
|
||||
|
||||
def _load_or_create_key(self) -> bytes:
|
||||
def _load_or_create_key(self, bootstrap_ok: bool) -> bytes:
|
||||
if self._key_path.exists():
|
||||
return self._key_path.read_bytes()
|
||||
if not bootstrap_ok:
|
||||
raise MissingCredentialKey(
|
||||
f"Fernet key file not found at {self._key_path}. "
|
||||
f"For first-time setup, set {_BOOTSTRAP_ENV_VAR}=1. "
|
||||
f"If this is a restored instance, restore the key file "
|
||||
f"from backup — generating a new one would make every "
|
||||
f"existing Credential row undecryptable."
|
||||
)
|
||||
log.warning(
|
||||
"Generating NEW Fernet credential key at %s. Any existing "
|
||||
"encrypted_blob rows in the DB will be undecryptable — "
|
||||
"re-upload each credential after this completes.",
|
||||
self._key_path,
|
||||
)
|
||||
parent = self._key_path.parent
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(parent, 0o700)
|
||||
|
||||
@@ -35,6 +35,7 @@ from .gallery_dl import (
|
||||
)
|
||||
from .importer import Importer
|
||||
from .patreon_resolver import resolve_campaign_id
|
||||
from .platforms import auth_type_for
|
||||
from .scheduler_service import set_platform_cooldown
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -177,6 +178,13 @@ class DownloadService:
|
||||
return {"status": "in_flight", "event_id": existing.id}
|
||||
if existing and existing.status == "pending":
|
||||
existing.status = "running"
|
||||
# Reset started_at on the pending→running transition so the
|
||||
# recovery sweep (DOWNLOAD_STALL_THRESHOLD_MINUTES, 30 min)
|
||||
# measures from real start, not from enqueue. On heavy-queue
|
||||
# days a freshly-promoted event whose original started_at
|
||||
# predated the cutoff would otherwise get swept mid-flight,
|
||||
# racing phase3's commit. Audit 2026-06-02.
|
||||
existing.started_at = datetime.now(UTC)
|
||||
await self.async_session.commit()
|
||||
event_id = existing.id
|
||||
else:
|
||||
@@ -187,7 +195,12 @@ class DownloadService:
|
||||
event_id = ev.id
|
||||
|
||||
artist = source.artist
|
||||
if source.platform in ("discord", "pixiv"):
|
||||
# Drive cookies-vs-token selection from the platform registry's
|
||||
# auth_type so a new 7th token-platform automatically picks the
|
||||
# right credential path. The hardcoded tuple here used to drift
|
||||
# out of sync with credential_service's auth_type_for(). Audit
|
||||
# 2026-06-02.
|
||||
if auth_type_for(source.platform) == "token":
|
||||
cookies_path = None
|
||||
auth_token = await self.cred_service.get_token(source.platform)
|
||||
else:
|
||||
@@ -308,6 +321,23 @@ class DownloadService:
|
||||
# failure. Don't flag the run as error; the file stays
|
||||
# on disk for operator inspection.
|
||||
import_summary["skipped"] += 1
|
||||
elif result.status == "failed":
|
||||
# Hard failure (today only: archive probe crash/timeout).
|
||||
# The original archive sits in /images/ as an orphan; the
|
||||
# filesystem scanner would re-import and re-crash on the
|
||||
# same file, so delete the source file and surface the
|
||||
# error in import_summary. Audit 2026-06-02.
|
||||
import_summary["errors"] += 1
|
||||
try:
|
||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
elif result.status == "refreshed":
|
||||
# Currently unreachable from attach_in_place (the download
|
||||
# path never runs in deep=True mode), but the importer's
|
||||
# ImportResult contract enumerates it. Treat the same as
|
||||
# 'attached' — work happened, no error. Audit 2026-06-02.
|
||||
import_summary["attached"] += 1
|
||||
else:
|
||||
import_summary["errors"] += 1
|
||||
|
||||
@@ -354,12 +384,26 @@ class DownloadService:
|
||||
# backfill run drained the queue (gallery-dl exited 0 + zero files
|
||||
# downloaded means there was nothing to fetch); otherwise decrement
|
||||
# the counter. Next tick falls back to tick mode once it hits 0.
|
||||
#
|
||||
# Audit 2026-06-02 gating: VALIDATION_FAILED also exits the
|
||||
# subprocess with return_code=0 and files_downloaded=0 (every
|
||||
# file was quarantined), which used to match the auto-complete
|
||||
# predicate exactly — zeroing the operator's armed budget on
|
||||
# the FIRST quarantine run instead of decrementing. Require
|
||||
# dl_result.success + no error_type so only genuinely-empty
|
||||
# successful runs drain the counter.
|
||||
backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
|
||||
if backfill_remaining > 0:
|
||||
src = (await self.async_session.execute(
|
||||
select(Source).where(Source.id == ctx["source_id"])
|
||||
)).scalar_one()
|
||||
if dl_result.return_code == 0 and dl_result.files_downloaded == 0:
|
||||
queue_drained = (
|
||||
dl_result.success
|
||||
and dl_result.error_type is None
|
||||
and dl_result.return_code == 0
|
||||
and dl_result.files_downloaded == 0
|
||||
)
|
||||
if queue_drained:
|
||||
src.backfill_runs_remaining = 0
|
||||
else:
|
||||
src.backfill_runs_remaining = max(0, backfill_remaining - 1)
|
||||
@@ -390,9 +434,15 @@ class DownloadService:
|
||||
if status == "ok":
|
||||
source.consecutive_failures = 0
|
||||
source.last_error = None
|
||||
# alembic 0032 — clear the failure-class chip on success.
|
||||
source.error_type = None
|
||||
elif status == "error":
|
||||
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
||||
source.last_error = error_message
|
||||
# alembic 0032 — stamp the failure-class so FailingSourcesCard
|
||||
# can render a colored chip and operators can bulk-triage
|
||||
# by error class without opening Logs per row.
|
||||
source.error_type = error_type
|
||||
if error_type == "rate_limited":
|
||||
await set_platform_cooldown(self.async_session, source.platform)
|
||||
elif status == "skipped":
|
||||
|
||||
@@ -374,10 +374,19 @@ class Importer:
|
||||
artist = self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist)
|
||||
sha = _sha256_of(source)
|
||||
existing = self.session.execute(
|
||||
select(PostAttachment).where(PostAttachment.sha256 == sha)
|
||||
).scalar_one_or_none()
|
||||
if existing is None:
|
||||
select_existing = select(PostAttachment).where(PostAttachment.sha256 == sha)
|
||||
existing = self.session.execute(select_existing).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
self.session.commit()
|
||||
return ImportResult(status="attached")
|
||||
# Savepoint + IntegrityError recovery — PostAttachment.sha256 is
|
||||
# UNIQUE, so two workers can both pass the SELECT and only the
|
||||
# second INSERT fails. Without savepoint, the outer transaction
|
||||
# poisons and the calling task crashes. attachments.store is
|
||||
# sha-addressed so both workers race to write the same target
|
||||
# path; shutil.copy2 + rename is idempotent. Audit 2026-06-02.
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
stored = self.attachments.store(source, sha)
|
||||
self.session.add(PostAttachment(
|
||||
post_id=post.id if post else None,
|
||||
@@ -390,10 +399,19 @@ class Importer:
|
||||
size_bytes=source.stat().st_size,
|
||||
))
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
# Lost the race — the other worker's row is canonical.
|
||||
self.session.execute(select_existing).scalar_one()
|
||||
self.session.commit()
|
||||
return ImportResult(status="attached")
|
||||
|
||||
def _import_archive(self, source: Path) -> ImportResult:
|
||||
def _import_archive(
|
||||
self, source: Path, *,
|
||||
artist: Artist | None = None,
|
||||
source_row: Source | None = None,
|
||||
) -> ImportResult:
|
||||
# Layer-3 isolation: bomb-size guard + integrity test in a
|
||||
# spawned child BEFORE extracting in this process. A
|
||||
# decompression bomb or a native-lib crash on a malformed
|
||||
@@ -401,6 +419,14 @@ class Importer:
|
||||
# instead of OOMing/segfaulting the import worker. extract_archive
|
||||
# is already fail-soft for plain exceptions, so this only adds
|
||||
# the hard-crash protection.
|
||||
#
|
||||
# Audit 2026-06-02: optional artist/source_row kwargs let the
|
||||
# download path thread its explicit subscription context
|
||||
# through instead of having _resolve_artist re-derive from
|
||||
# path-walk (which works by coincidence today because gallery-dl
|
||||
# lays files out under /images/<artist_slug>/...). Filesystem
|
||||
# import still calls bare _import_archive(source) and falls
|
||||
# back to the path-walk derivation as before.
|
||||
probe = safe_probe.probe_archive(source)
|
||||
if not probe.ok:
|
||||
if probe.crashed:
|
||||
@@ -412,24 +438,28 @@ class Importer:
|
||||
# still preserve the archive file itself as an attachment so
|
||||
# nothing silently vanishes, matching extract_archive's
|
||||
# fail-soft contract.
|
||||
artist = self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist)
|
||||
self._capture_attachment(source, post=post, artist=artist, resolved=True)
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use)
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist_use, resolved=True,
|
||||
)
|
||||
return ImportResult(status="attached")
|
||||
|
||||
artist = self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist)
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use)
|
||||
member_ids: list[int] = []
|
||||
with extract_archive(source) as members:
|
||||
for _name, member_path in members:
|
||||
if not is_supported(member_path):
|
||||
continue # non-media preserved via the stored archive
|
||||
res = self._import_media(member_path, source)
|
||||
res = self._import_media(
|
||||
member_path, source, explicit_source=source_row,
|
||||
)
|
||||
if res.status in ("imported", "superseded") and res.image_id:
|
||||
member_ids.append(res.image_id)
|
||||
# Preserve the archive itself (links to the same Post/Artist).
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist, resolved=True
|
||||
source, post=post, artist=artist_use, resolved=True
|
||||
)
|
||||
if member_ids:
|
||||
return ImportResult(
|
||||
@@ -439,7 +469,8 @@ class Importer:
|
||||
return ImportResult(status="attached")
|
||||
|
||||
def _import_media(
|
||||
self, source: Path, attribution_path: Path
|
||||
self, source: Path, attribution_path: Path,
|
||||
*, explicit_source: Source | None = None,
|
||||
) -> ImportResult:
|
||||
"""The media import pipeline (filters, dedup, copy, provenance).
|
||||
|
||||
@@ -586,7 +617,15 @@ class Importer:
|
||||
artist = self._attach_artist(record, artist_name)
|
||||
|
||||
# Sidecar provenance (best-effort; never fails the import).
|
||||
self._apply_sidecar(record, attribution_path, artist)
|
||||
# explicit_source lets the FC-3c download path bind the new
|
||||
# ImageProvenance row to its subscription Source instead of
|
||||
# having _apply_sidecar re-derive via _lookup_source_for_sidecar.
|
||||
# Audit 2026-06-02 — archive members extracted from a
|
||||
# subscription-downloaded zip previously lost subscription
|
||||
# linkage if the on-disk layout didn't match assumptions.
|
||||
self._apply_sidecar(
|
||||
record, attribution_path, artist, explicit_source=explicit_source,
|
||||
)
|
||||
|
||||
# Thumbnail is queued separately by the calling task; the importer
|
||||
# does not generate thumbnails inline so the import queue stays moving.
|
||||
@@ -673,7 +712,9 @@ class Importer:
|
||||
error="sidecar json is metadata, not content",
|
||||
)
|
||||
if is_archive(path):
|
||||
return self._import_archive(path)
|
||||
return self._import_archive(
|
||||
path, artist=artist, source_row=source,
|
||||
)
|
||||
if not is_supported(path):
|
||||
post = self._post_for_sidecar(path, artist) if artist else None
|
||||
return self._capture_attachment(
|
||||
@@ -749,7 +790,8 @@ class Importer:
|
||||
if rel == "smaller_exists":
|
||||
target = self.session.get(ImageRecord, match_id)
|
||||
self._supersede(
|
||||
target, path, sha, phash, width, height, new_path=path
|
||||
target, path, sha, phash, width, height,
|
||||
new_path=path, artist=artist, source_row=source,
|
||||
)
|
||||
return ImportResult(status="superseded", image_id=match_id)
|
||||
|
||||
@@ -944,6 +986,8 @@ class Importer:
|
||||
self, existing: ImageRecord, source: Path, sha: str,
|
||||
phash: str, width: int | None, height: int | None,
|
||||
*, new_path: Path | None = None,
|
||||
artist: Artist | None = None,
|
||||
source_row: Source | None = None,
|
||||
) -> None:
|
||||
"""Replace `existing`'s file with the larger `source`, keeping the
|
||||
row id (so tags/series/curation stay attached). ML is cleared so
|
||||
@@ -995,8 +1039,14 @@ class Importer:
|
||||
# _apply_sidecar resolves artist from the sidecar itself if the
|
||||
# existing row has none, and is internally guarded against
|
||||
# missing-or-malformed sidecars (silent return).
|
||||
# Audit 2026-06-02: thread artist/source_row from the
|
||||
# download-path caller (attach_in_place smaller_exists branch)
|
||||
# so the supersede preserves explicit subscription linkage
|
||||
# instead of re-deriving via path-walk.
|
||||
try:
|
||||
self._apply_sidecar(existing, source, None)
|
||||
self._apply_sidecar(
|
||||
existing, source, artist, explicit_source=source_row,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Don't unwind the supersede DB swap if sidecar parsing
|
||||
# blows up unexpectedly — the file replacement is the
|
||||
|
||||
@@ -19,7 +19,6 @@ from ...models import (
|
||||
TagReferenceEmbedding,
|
||||
)
|
||||
from ...models.tag import image_tag
|
||||
from .embedder import MODEL_VERSION as SIGLIP_VERSION
|
||||
|
||||
ELIGIBLE_KINDS = {
|
||||
TagKind.character,
|
||||
@@ -46,6 +45,21 @@ class CentroidService:
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
async def _model_version(self) -> str:
|
||||
"""Audit 2026-06-02: SigLIP model-version stamp comes from the
|
||||
DB row, not the env constant. tag_and_embed (tasks/ml.py:110)
|
||||
already reads from MLSettings.embedder_model_version, so by
|
||||
sourcing centroid stamps + drift checks from the same row, we
|
||||
eliminate the silent-drift case the audit flagged. env
|
||||
SIGLIP_MODEL_VERSION still drives which model embedder.py
|
||||
loads at runtime; the version stamp is purely the operator-
|
||||
controlled identifier."""
|
||||
return (
|
||||
await self.session.execute(
|
||||
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
async def recompute_for_tag(self, tag_id: int) -> bool:
|
||||
"""Recompute one tag's centroid. Returns True if a centroid was
|
||||
written, False if skipped (ineligible kind or too few members)."""
|
||||
@@ -69,19 +83,20 @@ class CentroidService:
|
||||
return False
|
||||
|
||||
centroid = np.mean(np.stack(embeddings), axis=0).astype(np.float32)
|
||||
model_version = await self._model_version()
|
||||
|
||||
stmt = insert(TagReferenceEmbedding).values(
|
||||
tag_id=tag_id,
|
||||
embedding=centroid.tolist(),
|
||||
reference_count=len(embeddings),
|
||||
model_version=SIGLIP_VERSION,
|
||||
model_version=model_version,
|
||||
)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["tag_id"],
|
||||
set_={
|
||||
"embedding": centroid.tolist(),
|
||||
"reference_count": len(embeddings),
|
||||
"model_version": SIGLIP_VERSION,
|
||||
"model_version": model_version,
|
||||
"updated_at": func.now(),
|
||||
},
|
||||
)
|
||||
@@ -92,6 +107,7 @@ class CentroidService:
|
||||
"""Tag ids whose centroid is stale: member count != reference_count,
|
||||
OR no centroid row, OR centroid built on a different SigLIP version.
|
||||
Only considers eligible-kind tags with embeddings present."""
|
||||
current_model_version = await self._model_version()
|
||||
member_counts = (
|
||||
select(
|
||||
image_tag.c.tag_id.label("tag_id"),
|
||||
@@ -116,7 +132,7 @@ class CentroidService:
|
||||
TagReferenceEmbedding.reference_count
|
||||
!= member_counts.c.members
|
||||
)
|
||||
| (TagReferenceEmbedding.model_version != SIGLIP_VERSION)
|
||||
| (TagReferenceEmbedding.model_version != current_model_version)
|
||||
)
|
||||
)
|
||||
return list((await self.session.execute(stmt)).scalars().all())
|
||||
|
||||
@@ -4,7 +4,7 @@ threshold-filtered, category-grouped, ranked suggestions for one image.
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import (
|
||||
@@ -16,6 +16,7 @@ from ...models import (
|
||||
from ...models.tag import image_tag
|
||||
from .aliases import AliasService
|
||||
from .centroids import CentroidService
|
||||
from .tag_name import normalize as normalize_tag_name
|
||||
from .tagger import SURFACED_CATEGORIES
|
||||
|
||||
|
||||
@@ -84,7 +85,12 @@ class SuggestionService:
|
||||
)
|
||||
|
||||
# --- Camie predictions ---
|
||||
candidates: list[tuple[str, str, float]] = []
|
||||
# candidates carry (raw_name, display_name, category, confidence).
|
||||
# raw_name = the booru-formatted vocab key, kept for alias_map
|
||||
# lookup since alias rows are hand-curated against raw keys.
|
||||
# display_name = normalize_tag_name(raw_name) — what the operator
|
||||
# sees AND what gets written to tag.name on Accept.
|
||||
candidates: list[tuple[str, str, str, float]] = []
|
||||
for name, p in predictions.items():
|
||||
category = p.get("category", "general")
|
||||
if category not in SURFACED_CATEGORIES:
|
||||
@@ -92,10 +98,14 @@ class SuggestionService:
|
||||
conf = float(p.get("confidence", 0.0))
|
||||
if conf < self._threshold_for(settings, category):
|
||||
continue
|
||||
candidates.append((name, category, conf))
|
||||
display = normalize_tag_name(name)
|
||||
if display is None:
|
||||
# emoticon / pure-punctuation vocab entry — drop entirely
|
||||
continue
|
||||
candidates.append((name, display, category, conf))
|
||||
|
||||
alias_map = await self.aliases.resolve_many(
|
||||
[(n, c) for n, c, _ in candidates]
|
||||
[(raw, c) for raw, _disp, c, _conf in candidates]
|
||||
)
|
||||
|
||||
merged: dict[object, Suggestion] = {}
|
||||
@@ -116,8 +126,8 @@ class SuggestionService:
|
||||
creates_new_tag=existing.creates_new_tag,
|
||||
)
|
||||
|
||||
for name, category, conf in candidates:
|
||||
canonical = alias_map.get((name, category))
|
||||
for raw, display, category, conf in candidates:
|
||||
canonical = alias_map.get((raw, category))
|
||||
if canonical is not None:
|
||||
if canonical.id in applied or canonical.id in rejected:
|
||||
continue
|
||||
@@ -133,9 +143,17 @@ class SuggestionService:
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Case-insensitive match on BOTH the raw camie key AND
|
||||
# the normalized form — covers legacy underscore-named
|
||||
# Tag rows accepted before normalization shipped, AND
|
||||
# any tag the operator created with the human form.
|
||||
existing_tag = (
|
||||
await self.session.execute(
|
||||
select(Tag).where(Tag.name == name)
|
||||
select(Tag).where(
|
||||
func.lower(Tag.name).in_(
|
||||
[raw.lower(), display.lower()]
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalars().first()
|
||||
if existing_tag is not None:
|
||||
@@ -157,10 +175,10 @@ class SuggestionService:
|
||||
)
|
||||
else:
|
||||
_merge(
|
||||
f"raw:{name}:{category}",
|
||||
f"raw:{display}:{category}",
|
||||
Suggestion(
|
||||
canonical_tag_id=None,
|
||||
display_name=name,
|
||||
display_name=display,
|
||||
category=category,
|
||||
score=conf,
|
||||
source="tagger",
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Camie vocabulary -> human-readable tag-name normalization.
|
||||
|
||||
Camie v2's ~57k tag vocabulary is booru-derived and arrives as raw
|
||||
strings like `uchiha_sasuke_(naruto)`, `#unicus_(idolmaster)`,
|
||||
`1000-nen_ikiteru_(vocaloid)`, or `:/`. We want the operator to see
|
||||
"Uchiha Sasuke", "Unicus", "1000-Nen Ikiteru", or to never see the
|
||||
emoticon at all — and we want the same clean string to be what lands
|
||||
in `tag.name` when the suggestion is accepted, so Accept matches the
|
||||
existing-tag convention (`tag_service.find_or_create`).
|
||||
|
||||
Rules (operator-approved 2026-06-03):
|
||||
1. Strip leading junk chars (#, ., +, ;, ~, _, whitespace)
|
||||
2. Drop trailing `_(disambiguator)` block(s), iteratively
|
||||
3. Strip wrapping single/double quotes (after disambig removal so
|
||||
`"foo_em_up"_(series)` -> `"foo_em_up"` -> `foo_em_up`)
|
||||
4. Replace remaining `_` with space; collapse runs of whitespace
|
||||
5. Add a space after any `:` (namespace:tag -> namespace: tag)
|
||||
6. Preserve hyphens (booru hyphens often carry meaning)
|
||||
7. Title-case each space-separated word (first character only —
|
||||
apostrophes, digits, hyphens stay)
|
||||
8. If no letters AND no digits remain, return None (drops emoticons
|
||||
like `:/` or `^_^`; preserves bare digit tags like `2005`)
|
||||
9. No surname/givenname swap — no reliable signal in the vocab
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
_LEADING_JUNK = re.compile(r"^[#.+;~_\s]+")
|
||||
_TRAILING_DISAMBIG = re.compile(r"_\([^)]*\)\s*$")
|
||||
_MULTISPACE = re.compile(r"\s+")
|
||||
_COLON_NOSPACE = re.compile(r":(?=\S)")
|
||||
_HAS_ALPHANUMERIC = re.compile(r"[A-Za-z0-9]")
|
||||
|
||||
|
||||
def _strip_wrapping_quotes(s: str) -> str:
|
||||
if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"):
|
||||
return s[1:-1]
|
||||
return s
|
||||
|
||||
|
||||
def _title_word(w: str) -> str:
|
||||
return w[:1].upper() + w[1:] if w else w
|
||||
|
||||
|
||||
def normalize(raw: str) -> str | None:
|
||||
"""Return the human-readable form of a raw Camie tag, or None if the
|
||||
string is junk (emoticon, empty after stripping)."""
|
||||
if not raw:
|
||||
return None
|
||||
s = _LEADING_JUNK.sub("", raw)
|
||||
while True:
|
||||
new = _TRAILING_DISAMBIG.sub("", s)
|
||||
if new == s:
|
||||
break
|
||||
s = new
|
||||
s = _strip_wrapping_quotes(s)
|
||||
s = s.replace("_", " ")
|
||||
s = _COLON_NOSPACE.sub(": ", s)
|
||||
s = _MULTISPACE.sub(" ", s).strip()
|
||||
if not s or not _HAS_ALPHANUMERIC.search(s):
|
||||
return None
|
||||
return " ".join(_title_word(w) for w in s.split(" "))
|
||||
@@ -59,6 +59,7 @@ class SourceRecord:
|
||||
config_overrides: dict | None
|
||||
last_checked_at: str | None
|
||||
last_error: str | None
|
||||
error_type: str | None
|
||||
check_interval_override: int | None
|
||||
consecutive_failures: int
|
||||
next_check_at: str | None
|
||||
@@ -76,6 +77,7 @@ class SourceRecord:
|
||||
"config_overrides": self.config_overrides,
|
||||
"last_checked_at": self.last_checked_at,
|
||||
"last_error": self.last_error,
|
||||
"error_type": self.error_type,
|
||||
"check_interval_override": self.check_interval_override,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"next_check_at": self.next_check_at,
|
||||
@@ -144,6 +146,7 @@ class SourceService:
|
||||
config_overrides=source.config_overrides,
|
||||
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
||||
last_error=source.last_error,
|
||||
error_type=source.error_type,
|
||||
check_interval_override=source.check_interval_override,
|
||||
consecutive_failures=source.consecutive_failures or 0,
|
||||
next_check_at=nxt.isoformat() if nxt else None,
|
||||
|
||||
@@ -5,6 +5,7 @@ from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import and_, case, exists, func, select, text, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Tag, TagKind, image_tag
|
||||
@@ -86,9 +87,12 @@ class TagService:
|
||||
f"fandom_id {fandom_id} does not reference a fandom tag"
|
||||
)
|
||||
|
||||
# Upsert via INSERT ... ON CONFLICT DO NOTHING. We can't use the
|
||||
# uniqueness index name directly (it's a partial coalesce-based
|
||||
# expression), so we re-select after insert.
|
||||
# Audit 2026-06-02: race-safe upsert via savepoint +
|
||||
# IntegrityError recovery. The partial uniqueness index on
|
||||
# (name, kind, COALESCE(fandom_id, -1)) catches concurrent
|
||||
# inserts; without the savepoint the outer transaction would
|
||||
# poison and the calling request crashes. Mirrors
|
||||
# importer._get_or_create.
|
||||
stmt = (
|
||||
select(Tag)
|
||||
.where(Tag.name == name)
|
||||
@@ -101,10 +105,16 @@ class TagService:
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
new_tag = Tag(name=name, kind=kind, fandom_id=fandom_id)
|
||||
self.session.add(new_tag)
|
||||
await self.session.flush()
|
||||
return new_tag
|
||||
sp = await self.session.begin_nested()
|
||||
try:
|
||||
new_tag = Tag(name=name, kind=kind, fandom_id=fandom_id)
|
||||
self.session.add(new_tag)
|
||||
await self.session.flush()
|
||||
await sp.commit()
|
||||
return new_tag
|
||||
except IntegrityError:
|
||||
await sp.rollback()
|
||||
return (await self.session.execute(stmt)).scalar_one()
|
||||
|
||||
async def autocomplete(
|
||||
self,
|
||||
|
||||
@@ -240,8 +240,11 @@ def prune_backups() -> dict:
|
||||
|
||||
Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
|
||||
Tagged rows (tag IS NOT NULL) are never pruned.
|
||||
Status='running' / 'restoring' rows are never pruned (recovery
|
||||
sweep from FC-3i handles those via task_run).
|
||||
Status='running' / 'restoring' rows are never pruned — the
|
||||
recover_stalled_backup_runs sweep flips truly-stuck ones to
|
||||
'error' first. (Earlier docstring claimed the FC-3i TaskRun sweep
|
||||
handled those, but TaskRun cleanup never touched BackupRun rows.
|
||||
Audit 2026-06-02 added the dedicated sweep.)
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
||||
|
||||
@@ -11,10 +11,13 @@ from sqlalchemy import Integer, and_, cast, delete, func, or_, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import (
|
||||
BackupRun,
|
||||
DownloadEvent,
|
||||
ImageRecord,
|
||||
ImportBatch,
|
||||
ImportSettings,
|
||||
ImportTask,
|
||||
LibraryAuditRun,
|
||||
Source,
|
||||
TaskRun,
|
||||
)
|
||||
@@ -55,6 +58,29 @@ FFPROBE_TIMEOUT_SECONDS = 10
|
||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
|
||||
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
||||
# > the entity's longest legitimate runtime (its task's time_limit + a
|
||||
# small buffer) so the sweep never flags in-flight work.
|
||||
#
|
||||
# Backups: images backup has time_limit=23400s (6.5h). 7h covers it
|
||||
# with a 30-min buffer; db backup at 12 min hard limit fits trivially.
|
||||
BACKUP_STALL_THRESHOLD_MINUTES = 7 * 60
|
||||
# Library audit: scan_library_for_rule has time_limit=7500s (2h5m).
|
||||
# 2h15m gives a 10-min buffer.
|
||||
LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135
|
||||
# Import batches finalize only after every child ImportTask hits a
|
||||
# terminal state. The recovery sweep targets the case where every
|
||||
# task is done but the batch never got its closing UPDATE
|
||||
# (orchestrator crashed at the wrong instant). 2h is well past any
|
||||
# realistic single-batch import.
|
||||
IMPORT_BATCH_STALL_THRESHOLD_MINUTES = 120
|
||||
|
||||
# Retention windows (terminal rows older than these get deleted by
|
||||
# the daily prune sweeps). 30 days = operator-flagged "useful for
|
||||
# triage for a few weeks, then noise."
|
||||
LIBRARY_AUDIT_KEEP_DAYS = 30
|
||||
IMPORT_BATCH_KEEP_DAYS = 30
|
||||
|
||||
# Overrides for recover_stalled_task_runs (the TaskRun 'running' sweep).
|
||||
# Tasks/queues that legitimately run longer than the default 5-min
|
||||
# threshold need their own larger value, else the sweep marks in-flight
|
||||
@@ -69,9 +95,24 @@ TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
# files); time_limit=2100.
|
||||
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"ml": 25,
|
||||
# Audit 2026-06-02 — maintenance/scan queues run tasks that
|
||||
# legitimately exceed the 5-min default (verify_integrity at 70m
|
||||
# hard, scan_directory at 70m hard, apply_allowlist_tags /
|
||||
# recompute_centroids / backfill_phash at 35m hard). 75 min lives
|
||||
# above the longest of those and the per-task overrides below
|
||||
# cover the outliers (backups, library audit).
|
||||
"maintenance": 75,
|
||||
"scan": 75,
|
||||
}
|
||||
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||
# Backup images runs hours, not minutes (6.5h hard limit). The
|
||||
# task-name override beats the queue's 75-min default so a
|
||||
# legitimately-running backup isn't flagged.
|
||||
"backend.app.tasks.backup.backup_images_task": 420,
|
||||
"backend.app.tasks.backup.restore_images_task": 420,
|
||||
# Library audit scans the full library — 2h hard limit.
|
||||
"backend.app.tasks.library_audit.scan_library_for_rule": 130,
|
||||
}
|
||||
|
||||
|
||||
@@ -360,7 +401,12 @@ def prune_task_runs() -> dict:
|
||||
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
||||
@celery.task(
|
||||
name="backend.app.tasks.maintenance.backfill_phash",
|
||||
# Audit 2026-06-02 — keyset-paginated phash recompute over the whole
|
||||
# library; legitimately runs >5 min on large libraries.
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def backfill_phash() -> int:
|
||||
"""Recompute phash for stored images that have none (imported before
|
||||
FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill,
|
||||
@@ -438,7 +484,13 @@ def _verify_one(path: Path, expected_sha: str, mime: str, sha_fn) -> str:
|
||||
return "failed_verification"
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.verify_integrity")
|
||||
@celery.task(
|
||||
name="backend.app.tasks.maintenance.verify_integrity",
|
||||
# Audit 2026-06-02 — full library sha256 + decode probe; on 100k-image
|
||||
# libraries this runs an hour or more. Match the maintenance queue's
|
||||
# recovery threshold (75 min) with 30s buffer below.
|
||||
soft_time_limit=3600, time_limit=4200,
|
||||
)
|
||||
def verify_integrity() -> int:
|
||||
"""Verify every ImageRecord file: sha256 recompute + decode/probe
|
||||
(PIL for images; ffprobe for videos). Writes integrity_status
|
||||
@@ -534,6 +586,156 @@ def recover_stalled_download_events() -> int:
|
||||
return events_recovered
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
|
||||
def recover_stalled_backup_runs() -> int:
|
||||
"""Flip BackupRun rows stuck in running/restoring past the hard limit
|
||||
to error. Audit 2026-06-02.
|
||||
|
||||
prune_backups (FC-3h) used to claim the FC-3i task_run sweep handled
|
||||
these — but that sweep only flips TaskRun rows, not the BackupRun
|
||||
artifact rows. A SIGKILL'd backup left BackupRun stuck forever
|
||||
(dashboard showed phantom in-flight backups, keep_last_n offset
|
||||
arithmetic skewed because zombies sat outside the ok/error window).
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES)
|
||||
msg = (
|
||||
f"stranded by recovery sweep (no terminal status after "
|
||||
f"{BACKUP_STALL_THRESHOLD_MINUTES // 60}h)"
|
||||
)
|
||||
with SessionLocal() as session:
|
||||
result = session.execute(
|
||||
update(BackupRun)
|
||||
.where(BackupRun.status.in_(["running", "restoring"]))
|
||||
.where(BackupRun.started_at < cutoff)
|
||||
.values(status="error", finished_at=now, error=msg)
|
||||
)
|
||||
session.commit()
|
||||
recovered = result.rowcount or 0
|
||||
if recovered:
|
||||
log.info("recover_stalled_backup_runs: recovered %d rows", recovered)
|
||||
return recovered
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_library_audit_runs")
|
||||
def recover_stalled_library_audit_runs() -> int:
|
||||
"""Flip LibraryAuditRun rows stuck in running past the hard limit
|
||||
to error. Audit 2026-06-02.
|
||||
|
||||
LibraryAuditRun.status='running' was protected by an exclusive
|
||||
guard in start_audit_run — a SIGKILL'd run would block all future
|
||||
audits until manual DB surgery. (The guard is now age-aware, but
|
||||
this sweep is what makes that work in practice.)
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(minutes=LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES)
|
||||
msg = (
|
||||
f"stranded by recovery sweep (no terminal status after "
|
||||
f"{LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES} min)"
|
||||
)
|
||||
with SessionLocal() as session:
|
||||
result = session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.status == "running")
|
||||
.where(LibraryAuditRun.started_at < cutoff)
|
||||
.values(status="error", finished_at=now, error=msg)
|
||||
)
|
||||
session.commit()
|
||||
recovered = result.rowcount or 0
|
||||
if recovered:
|
||||
log.info(
|
||||
"recover_stalled_library_audit_runs: recovered %d rows", recovered,
|
||||
)
|
||||
return recovered
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_import_batches")
|
||||
def recover_stalled_import_batches() -> int:
|
||||
"""Finalize ImportBatch rows stuck in running past the hard limit
|
||||
when NO outstanding ImportTask remains. Audit 2026-06-02.
|
||||
|
||||
A batch row finalizes only after every child task hits a terminal
|
||||
state. The orphan case: scanner crashed between the last task's
|
||||
completion and the batch's closing UPDATE. The
|
||||
`/api/import/status` route then surfaces the batch as 'active'
|
||||
indefinitely while `/api/system/stats` (which uses the same
|
||||
EXISTS predicate we apply below) correctly returns null.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(minutes=IMPORT_BATCH_STALL_THRESHOLD_MINUTES)
|
||||
with SessionLocal() as session:
|
||||
# Batches still 'running' past the cutoff whose tasks are all
|
||||
# terminal — there's no outstanding work, so flip the batch
|
||||
# too. Mirrors the EXISTS predicate the active-batch surfaces use.
|
||||
result = session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.status == "running")
|
||||
.where(ImportBatch.started_at < cutoff)
|
||||
.where(
|
||||
~select(ImportTask.id)
|
||||
.where(
|
||||
ImportTask.batch_id == ImportBatch.id,
|
||||
ImportTask.status.in_(["pending", "queued", "processing"]),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
.values(status="complete", finished_at=now)
|
||||
)
|
||||
session.commit()
|
||||
recovered = result.rowcount or 0
|
||||
if recovered:
|
||||
log.info(
|
||||
"recover_stalled_import_batches: finalized %d zombie batches",
|
||||
recovered,
|
||||
)
|
||||
return recovered
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.prune_library_audit_runs")
|
||||
def prune_library_audit_runs() -> int:
|
||||
"""Daily retention: delete terminal LibraryAuditRun rows older than
|
||||
LIBRARY_AUDIT_KEEP_DAYS. Never touches 'running'. Audit 2026-06-02.
|
||||
|
||||
Audit rows carry matched_ids JSONB blobs that can hold tens of
|
||||
thousands of ids; without retention these accumulate.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cutoff = datetime.now(UTC) - timedelta(days=LIBRARY_AUDIT_KEEP_DAYS)
|
||||
with SessionLocal() as session:
|
||||
result = session.execute(
|
||||
delete(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.status.in_(["ready", "applied", "cancelled", "error"]))
|
||||
.where(LibraryAuditRun.finished_at < cutoff)
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.prune_import_batches")
|
||||
def prune_import_batches() -> int:
|
||||
"""Daily retention: delete terminal ImportBatch rows older than
|
||||
IMPORT_BATCH_KEEP_DAYS. Cascade-deletes child ImportTask rows via
|
||||
the model relationship. Never touches 'running'. Audit 2026-06-02.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cutoff = datetime.now(UTC) - timedelta(days=IMPORT_BATCH_KEEP_DAYS)
|
||||
with SessionLocal() as session:
|
||||
# ORM-level delete here (not Core delete) so the
|
||||
# ImportBatch->tasks cascade fires; Core delete would skip it.
|
||||
old_batches = session.execute(
|
||||
select(ImportBatch)
|
||||
.where(ImportBatch.status.in_(["complete", "cancelled"]))
|
||||
.where(ImportBatch.finished_at < cutoff)
|
||||
).scalars().all()
|
||||
for batch in old_batches:
|
||||
session.delete(batch)
|
||||
session.commit()
|
||||
return len(old_batches)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events")
|
||||
def cleanup_old_download_events() -> int:
|
||||
"""FC-3d: delete terminal DownloadEvent rows older than the configured
|
||||
|
||||
+15
-2
@@ -212,7 +212,14 @@ def backfill(self) -> int:
|
||||
return enqueued
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.apply_allowlist_tags", bind=True)
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.apply_allowlist_tags",
|
||||
bind=True,
|
||||
# Audit 2026-06-02 — the full-sweep mode (neither tag_id nor image_id)
|
||||
# is O(images × allowlist) and legitimately runs >5 min on large
|
||||
# libraries. Cap matches the maintenance queue's recovery threshold.
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def apply_allowlist_tags(self, tag_id: int | None = None,
|
||||
image_id: int | None = None) -> int:
|
||||
"""Retroactively apply allowlisted tags.
|
||||
@@ -341,7 +348,13 @@ def recompute_centroid(self, tag_id: int) -> bool:
|
||||
return asyncio.run(_run())
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.recompute_centroids", bind=True)
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.recompute_centroids",
|
||||
bind=True,
|
||||
# Audit 2026-06-02 — drifted-centroid rebuild over potentially
|
||||
# hundreds of tags.
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def recompute_centroids(self) -> int:
|
||||
"""Daily: find drifted centroids, enqueue recompute_centroid for each."""
|
||||
import asyncio
|
||||
|
||||
@@ -35,7 +35,15 @@ def _iter_import_files(import_root: Path):
|
||||
yield entry
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.scan.scan_directory", bind=True)
|
||||
@celery.task(
|
||||
name="backend.app.tasks.scan.scan_directory",
|
||||
bind=True,
|
||||
# Audit 2026-06-02 — large libraries make the scan legitimately long.
|
||||
# Hard cap at 70 min so the corresponding QUEUE_STUCK_THRESHOLD_MINUTES
|
||||
# ("scan") of 75 min always wins; soft limit gives the task a clean
|
||||
# exit window before SIGKILL.
|
||||
soft_time_limit=3600, time_limit=4200,
|
||||
)
|
||||
def scan_directory(self, triggered_by: str = "manual",
|
||||
mode: str = "quick") -> int:
|
||||
"""Walks the import root and creates ImportTasks. `mode` is 'quick'
|
||||
|
||||
@@ -194,9 +194,20 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
if (platform.authType === 'cookies') {
|
||||
const cookies = await extractCookiesForPlatform(key);
|
||||
if (cookies.length === 0) return { error: 'No cookies found — log in first.' };
|
||||
// Verify the captured cookies are actually live BEFORE
|
||||
// uploading. Skips upload on confirmed-stale sessions so we
|
||||
// don't overwrite FC-side credentials with garbage. Platforms
|
||||
// without a verify config (verify.ok === null) fall through
|
||||
// to upload as before.
|
||||
const v = await verifyCookiesForPlatform(key);
|
||||
if (v.ok === false) {
|
||||
return {
|
||||
error: `Captured ${cookies.length} ${platform.name} cookies but they don't appear authenticated (${v.reason}). Log in again in this browser, then retry.`,
|
||||
};
|
||||
}
|
||||
const data = toNetscapeFormat(cookies);
|
||||
await api.uploadCredentials(key, 'cookies', data);
|
||||
return { success: true, cookieCount: cookies.length };
|
||||
return { success: true, cookieCount: cookies.length, verified: v.ok === true };
|
||||
}
|
||||
if (key === 'discord') {
|
||||
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
||||
@@ -229,8 +240,13 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
results[key] = { skipped: true, reason: 'no cookies' };
|
||||
continue;
|
||||
}
|
||||
const v = await verifyCookiesForPlatform(key);
|
||||
if (v.ok === false) {
|
||||
results[key] = { error: `verify failed: ${v.reason}` };
|
||||
continue;
|
||||
}
|
||||
await api.uploadCredentials(key, 'cookies', toNetscapeFormat(cookies));
|
||||
results[key] = { success: true, cookieCount: cookies.length };
|
||||
results[key] = { success: true, cookieCount: cookies.length, verified: v.ok === true };
|
||||
} catch (e) {
|
||||
results[key] = { error: e.message };
|
||||
}
|
||||
|
||||
@@ -76,3 +76,38 @@ async function getCookieCount(platformKey) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify cookies are live by hitting an authenticated endpoint with the
|
||||
* browser's current cookie jar. Returns:
|
||||
* { ok: true, status } — verified
|
||||
* { ok: false, status, reason } — endpoint said we're not logged in
|
||||
* { ok: null, reason } — no verify config for this platform; caller
|
||||
* should treat as "verify not available,
|
||||
* proceed with upload"
|
||||
*
|
||||
* Implementation note: extensions with `host_permissions` for the target
|
||||
* domain get the user's cookies auto-attached to fetch() — same set
|
||||
* gallery-dl will later use on the backend.
|
||||
*/
|
||||
async function verifyCookiesForPlatform(platformKey) {
|
||||
const platform = PLATFORMS[platformKey];
|
||||
if (!platform) return { ok: false, reason: `Unknown platform: ${platformKey}` };
|
||||
if (!platform.verify) return { ok: null, reason: 'verify-not-configured' };
|
||||
|
||||
const { url, method, okStatuses } = platform.verify;
|
||||
let resp;
|
||||
try {
|
||||
resp = await fetch(url, { method, credentials: 'include', cache: 'no-store' });
|
||||
} catch (e) {
|
||||
return { ok: false, reason: `Verify request failed: ${e.message}` };
|
||||
}
|
||||
if (okStatuses.includes(resp.status)) {
|
||||
return { ok: true, status: resp.status };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
status: resp.status,
|
||||
reason: `${url} returned HTTP ${resp.status} — session looks stale or logged out`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -13,6 +13,13 @@ const PLATFORMS = {
|
||||
authType: 'cookies',
|
||||
color: '#FF424D',
|
||||
urlPattern: /^https?:\/\/(www\.)?patreon\.com/,
|
||||
// Patreon's `/api/current_user` returns 200 + the logged-in user
|
||||
// when authenticated, 401 otherwise. Cheapest definitive check.
|
||||
verify: {
|
||||
url: 'https://www.patreon.com/api/current_user',
|
||||
method: 'GET',
|
||||
okStatuses: [200],
|
||||
},
|
||||
},
|
||||
subscribestar: {
|
||||
name: 'SubscribeStar',
|
||||
@@ -26,6 +33,9 @@ const PLATFORMS = {
|
||||
authType: 'cookies',
|
||||
color: '#FFD700',
|
||||
urlPattern: /^https?:\/\/(www\.)?subscribestar\.(com|adult)/,
|
||||
// No known stable auth-required endpoint that returns a definitive
|
||||
// status code; skipping verify so we don't false-positive-fail
|
||||
// good cookies. Operator can add later if a clean endpoint surfaces.
|
||||
},
|
||||
hentaifoundry: {
|
||||
name: 'Hentai Foundry',
|
||||
@@ -33,6 +43,14 @@ const PLATFORMS = {
|
||||
authType: 'cookies',
|
||||
color: '#9C27B0',
|
||||
urlPattern: /^https?:\/\/(www\.)?hentai-foundry\.com/,
|
||||
// Mirror gallery-dl's _init_site_filters: HEAD on `?enterAgree=1`.
|
||||
// Logged in → 200, logged out → 401. Catches the exact failure mode
|
||||
// the backend extractor would hit later.
|
||||
verify: {
|
||||
url: 'https://www.hentai-foundry.com/?enterAgree=1',
|
||||
method: 'HEAD',
|
||||
okStatuses: [200],
|
||||
},
|
||||
},
|
||||
discord: {
|
||||
name: 'Discord',
|
||||
@@ -56,6 +74,9 @@ const PLATFORMS = {
|
||||
authType: 'cookies',
|
||||
color: '#05CC47',
|
||||
urlPattern: /^https?:\/\/(www\.)?deviantart\.com/,
|
||||
// DA's logged-in-only endpoints sit behind their internal _napi
|
||||
// namespace which shifts; skipping verify until a stable check
|
||||
// surfaces. Same posture as SubscribeStar.
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "FabledCurator",
|
||||
"version": "1.0.6",
|
||||
"version": "1.0.7",
|
||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||
|
||||
"browser_specific_settings": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fabledcurator-extension",
|
||||
"version": "1.0.6",
|
||||
"version": "1.0.7",
|
||||
"private": true,
|
||||
"description": "Firefox extension for FabledCurator",
|
||||
"scripts": {
|
||||
|
||||
@@ -132,8 +132,9 @@ async function exportPlatformCookies(key, card) {
|
||||
if (r.error) showError(r.error);
|
||||
else {
|
||||
const n = r.cookieCount ?? null;
|
||||
const verifiedSuffix = r.verified ? ' (verified ✓)' : '';
|
||||
const msg = n !== null
|
||||
? `${PLATFORMS[key].name}: ${n} cookies exported`
|
||||
? `${PLATFORMS[key].name}: ${n} cookies exported${verifiedSuffix}`
|
||||
: `${PLATFORMS[key].name}: token exported`;
|
||||
showSuccess(msg);
|
||||
await loadPlatformStatus();
|
||||
|
||||
+14
-1
@@ -9,7 +9,9 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { onMounted, ref, watch } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
|
||||
import AppShell from './components/AppShell.vue'
|
||||
import AppSnackbar from './components/AppSnackbar.vue'
|
||||
import ImageViewer from './components/modal/ImageViewer.vue'
|
||||
@@ -17,9 +19,20 @@ import { useModalStore } from './stores/modal.js'
|
||||
|
||||
const modal = useModalStore()
|
||||
const snackbar = ref(null)
|
||||
const route = useRoute()
|
||||
|
||||
onMounted(() => {
|
||||
// Expose snackbar via a simple global so stores can call it without props.
|
||||
window.__fcToast = (opts) => snackbar.value?.open(opts)
|
||||
})
|
||||
|
||||
// Audit 2026-06-02: the modal is an overlay, not a page. When the
|
||||
// route changes (RouterLink inside the modal, history back/forward,
|
||||
// programmatic push from any view), close the modal so it doesn't
|
||||
// hover over a different route. Watching route.name (not the path)
|
||||
// keeps within-route nav like /artist/foo → /artist/bar from
|
||||
// dismissing the modal mid-browse.
|
||||
watch(() => route.name, () => {
|
||||
if (modal.isOpen) modal.close()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -11,9 +11,6 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
|
||||
import { useArtistStore } from '../../stores/artist.js'
|
||||
import { useModalStore } from '../../stores/modal.js'
|
||||
import MasonryGrid from '../discovery/MasonryGrid.vue'
|
||||
@@ -24,22 +21,9 @@ const props = defineProps({
|
||||
|
||||
const store = useArtistStore()
|
||||
const modal = useModalStore()
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
||||
onMounted(() => {
|
||||
const initial = parseInt(route.query.image, 10)
|
||||
if (!isNaN(initial)) modal.open(initial)
|
||||
})
|
||||
|
||||
watch(() => route.query.image, (q) => {
|
||||
const id = parseInt(q, 10)
|
||||
if (!isNaN(id) && id !== modal.currentImageId) modal.open(id)
|
||||
else if (isNaN(id) && modal.currentImageId !== null) modal.close()
|
||||
})
|
||||
|
||||
function openImage (id) {
|
||||
router.push({ query: { ...route.query, image: id } })
|
||||
modal.open(id)
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
@@ -124,10 +124,16 @@ async function onRetry() {
|
||||
if (!props.event.source_id) return
|
||||
retrying.value = true
|
||||
try {
|
||||
await sourcesStore.checkNow(props.event.source_id)
|
||||
toast({
|
||||
text: `Source check re-queued`, type: 'success',
|
||||
})
|
||||
const body = await sourcesStore.checkNow(props.event.source_id)
|
||||
// Audit 2026-06-02: the previous handler unconditionally toasted
|
||||
// "re-queued" even when the platform was in cooldown (202 +
|
||||
// status='deferred'). Operator thought work was in flight when
|
||||
// nothing was actually enqueued.
|
||||
if (body?.status === 'deferred') {
|
||||
toast({ text: 'Retry deferred — platform in cooldown', type: 'info' })
|
||||
} else {
|
||||
toast({ text: 'Source check re-queued', type: 'success' })
|
||||
}
|
||||
} catch (e) {
|
||||
const isInFlight = !!e?.body?.download_event_id
|
||||
toast({
|
||||
|
||||
@@ -19,25 +19,34 @@
|
||||
>
|
||||
Accept
|
||||
</v-btn>
|
||||
<v-menu>
|
||||
<template #activator="{ props }">
|
||||
<v-btn
|
||||
class="fc-suggestion__menu"
|
||||
icon="mdi-dots-vertical" size="small"
|
||||
variant="outlined" density="compact"
|
||||
:aria-label="`More actions for ${suggestion.display_name}`"
|
||||
v-bind="props"
|
||||
/>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item @click="$emit('alias', suggestion)">
|
||||
<v-list-item-title>Treat as alias for…</v-list-item-title>
|
||||
</v-list-item>
|
||||
<v-list-item @click="$emit('dismiss', suggestion)">
|
||||
<v-list-item-title>Dismiss for this image</v-list-item-title>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
<!-- Operator-flagged 2026-06-02: the kebab menu wasn't opening.
|
||||
Wrapping in a <span @click.stop> matches the TagPanel chip
|
||||
fix — even though there's no parent click capture here today,
|
||||
the wrap is harmless and keeps both kebabs on the same
|
||||
pattern. Click bubbles from the v-btn → opens menu via
|
||||
activator props → bubble continues to span → stopPropagation
|
||||
halts it. -->
|
||||
<span class="fc-suggestion__menu-wrap" @click.stop>
|
||||
<v-menu>
|
||||
<template #activator="{ props }">
|
||||
<v-btn
|
||||
class="fc-suggestion__menu"
|
||||
icon="mdi-dots-vertical" size="small"
|
||||
variant="outlined" density="compact"
|
||||
:aria-label="`More actions for ${suggestion.display_name}`"
|
||||
v-bind="props"
|
||||
/>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item @click="$emit('alias', suggestion)">
|
||||
<v-list-item-title>Treat as alias for…</v-list-item-title>
|
||||
</v-list-item>
|
||||
<v-list-item @click="$emit('dismiss', suggestion)">
|
||||
<v-list-item-title>Dismiss for this image</v-list-item-title>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -90,6 +99,11 @@ const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
||||
.fc-suggestion__accept :deep(.v-btn__content) {
|
||||
font-size: 12px; letter-spacing: 0.02em;
|
||||
}
|
||||
.fc-suggestion__menu-wrap {
|
||||
flex: 0 0 auto;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
}
|
||||
.fc-suggestion__menu {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
@@ -10,19 +10,27 @@
|
||||
>
|
||||
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
||||
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
||||
<v-menu>
|
||||
<template #activator="{ props: mp }">
|
||||
<v-icon
|
||||
v-bind="mp" size="x-small" class="ml-1"
|
||||
icon="mdi-dots-vertical" @click.stop
|
||||
/>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item @click="openRename(tag)">
|
||||
<v-list-item-title>Rename…</v-list-item-title>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
<!-- Operator-flagged 2026-06-02: the previous activator had
|
||||
`@click.stop` directly on the v-icon, which silently
|
||||
overrode Vuetify's onClick from `v-bind="mp"` — the menu
|
||||
never opened. Now the v-icon receives the activator
|
||||
onClick cleanly, and the wrapping span absorbs the
|
||||
bubbled click so the chip's close button isn't tripped. -->
|
||||
<span class="kebab-wrap" @click.stop>
|
||||
<v-menu>
|
||||
<template #activator="{ props: mp }">
|
||||
<v-icon
|
||||
v-bind="mp" size="x-small" class="ml-1"
|
||||
icon="mdi-dots-vertical"
|
||||
/>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item @click="openRename(tag)">
|
||||
<v-list-item-title>Rename…</v-list-item-title>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</span>
|
||||
</v-chip>
|
||||
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
||||
</div>
|
||||
@@ -113,4 +121,5 @@ async function onRenamed() {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||
.kebab-wrap { display: inline-flex; align-items: center; }
|
||||
</style>
|
||||
|
||||
@@ -35,7 +35,7 @@ import { computed } from 'vue'
|
||||
const props = defineProps({
|
||||
queues: { type: Object, default: null }, // store.queues
|
||||
workers: { type: Object, default: null }, // store.workers
|
||||
recentMinute: { type: Array, default: () => [] }, // store.recentMinute
|
||||
recentRuns: { type: Array, default: () => [] }, // store.recentRuns
|
||||
compact: { type: Boolean, default: false },
|
||||
})
|
||||
|
||||
@@ -80,7 +80,7 @@ function activeCount(name) {
|
||||
|
||||
const recentByQueue = computed(() => {
|
||||
const out = {}
|
||||
for (const r of props.recentMinute) {
|
||||
for (const r of props.recentRuns) {
|
||||
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
|
||||
if (r.status === 'ok') out[r.queue].ok++
|
||||
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
<QueuesTable
|
||||
:queues="store.queues"
|
||||
:workers="store.workers"
|
||||
:recent-minute="store.recentMinute"
|
||||
:recent-runs="store.recentRuns"
|
||||
compact
|
||||
/>
|
||||
</v-card-text>
|
||||
@@ -47,7 +47,7 @@ function pollOnce() {
|
||||
if (document.hidden) return
|
||||
store.loadQueues()
|
||||
store.loadWorkers()
|
||||
store.loadRecentMinute()
|
||||
store.loadRecentRuns()
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<QueuesTable
|
||||
:queues="store.queues"
|
||||
:workers="store.workers"
|
||||
:recent-minute="store.recentMinute"
|
||||
:recent-runs="store.recentRuns"
|
||||
/>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
@@ -202,7 +202,7 @@ function pollQueues() {
|
||||
if (document.hidden) return
|
||||
store.loadQueues()
|
||||
store.loadWorkers()
|
||||
store.loadRecentMinute()
|
||||
store.loadRecentRuns()
|
||||
}
|
||||
function pollFailures() {
|
||||
if (document.hidden) return
|
||||
|
||||
@@ -25,6 +25,15 @@
|
||||
<v-chip size="x-small" color="error" variant="flat" label class="fc-fail__count">
|
||||
{{ s.consecutive_failures }}× failed
|
||||
</v-chip>
|
||||
<v-chip
|
||||
v-if="s.error_type"
|
||||
size="x-small" variant="outlined" label
|
||||
:color="errorTypeColor(s.error_type)"
|
||||
class="fc-fail__class"
|
||||
:title="errorTypeHint(s.error_type)"
|
||||
>
|
||||
{{ s.error_type }}
|
||||
</v-chip>
|
||||
<span class="fc-fail__err" :title="s.last_error || ''">
|
||||
{{ s.last_error || 'no error message recorded' }}
|
||||
</span>
|
||||
@@ -66,6 +75,42 @@ const open = ref(true)
|
||||
// Per-row loading flag so the spinner lives on the row whose Logs
|
||||
// button was clicked, not on every row.
|
||||
const logLoadingIds = ref(new Set())
|
||||
|
||||
// Audit 2026-06-02: surface the ErrorType taxonomy as a colored chip
|
||||
// next to the consecutive-failures count so operators can bulk-triage
|
||||
// by error class. Color reflects "what to do next":
|
||||
// warning (yellow) — auth/cookie issue: operator should rotate
|
||||
// info (blue) — backend-paced (cooldown / rate limit / timeout)
|
||||
// error (red) — likely terminal without operator intervention
|
||||
const ERROR_TYPE_COLOR = {
|
||||
auth_error: 'warning',
|
||||
rate_limited: 'info',
|
||||
timeout: 'info',
|
||||
network_error: 'info',
|
||||
not_found: 'error',
|
||||
access_denied: 'error',
|
||||
validation_failed: 'error',
|
||||
unsupported_url: 'error',
|
||||
http_error: 'error',
|
||||
unknown_error: 'error',
|
||||
partial: 'info',
|
||||
tier_limited: 'info',
|
||||
no_new_content: 'info',
|
||||
}
|
||||
const ERROR_TYPE_HINT = {
|
||||
auth_error: 'Cookies likely expired — re-upload in Credentials.',
|
||||
rate_limited: 'Platform-wide cooldown active. Will retry after it expires.',
|
||||
timeout: 'Subprocess exceeded its time budget. Often retries cleanly.',
|
||||
network_error: 'Transient network issue. Will retry on next tick.',
|
||||
not_found: 'URL 404 — creator may have renamed or deleted.',
|
||||
access_denied: 'Subscription tier may not grant this content.',
|
||||
validation_failed: 'Downloaded files were quarantined by the validator.',
|
||||
http_error: 'Generic HTTP error — see Logs.',
|
||||
unsupported_url: 'gallery-dl does not support this URL pattern.',
|
||||
unknown_error: 'Could not classify — see Logs.',
|
||||
}
|
||||
function errorTypeColor(t) { return ERROR_TYPE_COLOR[t] || 'error' }
|
||||
function errorTypeHint(t) { return ERROR_TYPE_HINT[t] || '' }
|
||||
async function onViewLogs(s) {
|
||||
if (logLoadingIds.value.has(s.id)) return
|
||||
logLoadingIds.value = new Set(logLoadingIds.value).add(s.id)
|
||||
@@ -115,6 +160,7 @@ async function onViewLogs(s) {
|
||||
}
|
||||
.fc-fail__artist { font-weight: 600; white-space: nowrap; }
|
||||
.fc-fail__count { flex: 0 0 auto; }
|
||||
.fc-fail__class { flex: 0 0 auto; }
|
||||
.fc-fail__err {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-size: 0.8rem;
|
||||
|
||||
@@ -412,6 +412,17 @@ function onArtistCreated(artist) {
|
||||
async function onCheck(source) {
|
||||
try {
|
||||
const body = await store.checkNow(source.id)
|
||||
// Audit 2026-06-02: /api/sources/<id>/check returns 202 with
|
||||
// `{status:'deferred', cooldown_until}` when the platform is in
|
||||
// cooldown — the previous handler treated this as success and
|
||||
// toasted "event #undefined", masking that nothing was enqueued.
|
||||
if (body?.status === 'deferred') {
|
||||
toast({
|
||||
text: 'Check deferred — platform in cooldown',
|
||||
type: 'info',
|
||||
})
|
||||
return
|
||||
}
|
||||
toast({
|
||||
text: `Check enqueued (event #${body.download_event_id})`,
|
||||
type: 'success',
|
||||
@@ -465,17 +476,23 @@ async function onBackfill(source) {
|
||||
async function checkAll(group) {
|
||||
let ok = 0
|
||||
let conflict = 0
|
||||
let deferred = 0
|
||||
for (const s of group.sources) {
|
||||
if (!s.enabled) continue
|
||||
try {
|
||||
await store.checkNow(s.id)
|
||||
ok += 1
|
||||
const body = await store.checkNow(s.id)
|
||||
// Audit 2026-06-02: deferred (202 + cooldown_until) used to be
|
||||
// counted as queued, inflating the success tally and hiding
|
||||
// that the cooldown actually held the work back.
|
||||
if (body?.status === 'deferred') deferred += 1
|
||||
else ok += 1
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) conflict += 1
|
||||
}
|
||||
}
|
||||
const parts = []
|
||||
if (ok) parts.push(`${ok} queued`)
|
||||
if (deferred) parts.push(`${deferred} deferred (cooldown)`)
|
||||
if (conflict) parts.push(`${conflict} already running`)
|
||||
toast({
|
||||
text: parts.join(', ') || 'Nothing to check (no enabled sources)',
|
||||
|
||||
@@ -155,6 +155,19 @@ export const useModalStore = defineStore('modal', () => {
|
||||
const imageId = currentImageId.value
|
||||
if (!imageId) return
|
||||
const tag = await api.post('/api/tags', { body: { name, kind, fandom_id } })
|
||||
// Audit 2026-06-02: a kind='fandom' created here used to be
|
||||
// invisible to FandomPicker until a full page reload — its load
|
||||
// gates on fandomCache.length, so a non-empty cache skips the
|
||||
// refetch and the new fandom never appears. Push it into the
|
||||
// cache directly so the next open sees it.
|
||||
if (kind === 'fandom') {
|
||||
const { useTagStore } = await import('./tags.js')
|
||||
const tagStore = useTagStore()
|
||||
tagStore.fandomCache.push({
|
||||
id: tag.id, name: tag.name, kind: 'fandom',
|
||||
fandom_id: null, fandom_name: null, image_count: 0,
|
||||
})
|
||||
}
|
||||
if (currentImageId.value !== imageId) return // navigated away
|
||||
await addExistingTag(tag.id)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||
// Live polled state.
|
||||
const queues = ref(null) // { queues: {name: depth|null}, fetched_at }
|
||||
const workers = ref(null) // { workers: {hostname: {...}}, fetched_at }
|
||||
const recentMinute = ref([]) // last-60s rows (for Overview summary)
|
||||
const recentRuns = ref([]) // last-60s rows (for Overview summary)
|
||||
const failures = ref(null) // { recent, count_by_type, since }
|
||||
|
||||
// Paginated runs (Activity tab "All recent activity" pane).
|
||||
@@ -45,14 +45,14 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function loadRecentMinute() {
|
||||
async function loadRecentRuns() {
|
||||
// Used by the Overview summary card: pull last 60s of runs to compute
|
||||
// per-queue ok/err counts. One call covers all queues; UI groups.
|
||||
try {
|
||||
const body = await api.get('/api/system/activity/runs', {
|
||||
params: { limit: 200 },
|
||||
})
|
||||
recentMinute.value = body.runs || []
|
||||
recentRuns.value = body.runs || []
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
}
|
||||
@@ -106,10 +106,10 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||
}
|
||||
|
||||
return {
|
||||
queues, workers, recentMinute, failures, summary,
|
||||
queues, workers, recentRuns, failures, summary,
|
||||
runs, runsCursor, runsHasMore, runsFilter,
|
||||
loading, lastError,
|
||||
loadQueues, loadWorkers, loadRecentMinute,
|
||||
loadQueues, loadWorkers, loadRecentRuns,
|
||||
loadRuns, loadFailures, loadSummary, setFilter,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
|
||||
<script setup>
|
||||
import { onMounted, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { useRoute } from 'vue-router'
|
||||
import { useGalleryStore } from '../stores/gallery.js'
|
||||
import { useModalStore } from '../stores/modal.js'
|
||||
import GalleryGrid from '../components/gallery/GalleryGrid.vue'
|
||||
@@ -37,7 +37,6 @@ import { useGallerySelectionStore } from '../stores/gallerySelection.js'
|
||||
const store = useGalleryStore()
|
||||
const modal = useModalStore()
|
||||
const sel = useGallerySelectionStore()
|
||||
const router = useRouter()
|
||||
const route = useRoute()
|
||||
|
||||
onMounted(async () => {
|
||||
@@ -47,9 +46,6 @@ onMounted(async () => {
|
||||
else if (!isNaN(tagId)) store.setTagFilter(tagId)
|
||||
await store.loadInitial()
|
||||
await store.loadTimeline()
|
||||
// Open modal if URL has ?image=N
|
||||
const initial = parseInt(route.query.image, 10)
|
||||
if (!isNaN(initial)) modal.open(initial)
|
||||
})
|
||||
|
||||
watch(() => route.query.tag_id, (q) => {
|
||||
@@ -64,19 +60,8 @@ watch(() => route.query.post_id, (q) => {
|
||||
store.setPostFilter(isNaN(postId) ? null : postId)
|
||||
})
|
||||
|
||||
watch(() => route.query.image, (q) => {
|
||||
const id = parseInt(q, 10)
|
||||
if (!isNaN(id) && id !== modal.currentImageId) modal.open(id)
|
||||
else if (isNaN(id) && modal.currentImageId !== null) modal.close()
|
||||
})
|
||||
|
||||
function openImage(id) {
|
||||
router.push({ query: { ...route.query, image: id } })
|
||||
}
|
||||
function closeImage() {
|
||||
const q = { ...route.query }
|
||||
delete q.image
|
||||
router.push({ query: q })
|
||||
modal.open(id)
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
+14
-7
@@ -8,18 +8,25 @@ migration code paths.
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
# Audit 2026-06-02: CredentialCrypto now refuses to auto-generate a
|
||||
# Fernet key without explicit opt-in (production safety against silent
|
||||
# key regeneration on partial restore). The test environment never has
|
||||
# a pre-seeded key file, so set the bootstrap flag here before any
|
||||
# create_app() / CredentialCrypto() import path fires.
|
||||
os.environ.setdefault("CURATOR_BOOTSTRAP_NEW_KEY", "1")
|
||||
|
||||
import pytest # noqa: E402
|
||||
import pytest_asyncio # noqa: E402
|
||||
from sqlalchemy import create_engine # noqa: E402
|
||||
from sqlalchemy.ext.asyncio import ( # noqa: E402
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.orm import sessionmaker # noqa: E402
|
||||
|
||||
from backend.app import create_app
|
||||
from backend.app.models import Base
|
||||
from backend.app import create_app # noqa: E402
|
||||
from backend.app.models import Base # noqa: E402
|
||||
|
||||
|
||||
def _async_database_url() -> str:
|
||||
|
||||
@@ -19,9 +19,10 @@ async def test_get_and_patch_settings(client):
|
||||
resp = await client.get("/api/ml/settings")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
# Default lowered 0.95 → 0.50 on 2026-06-01 (alembic 0029) — 0.95
|
||||
# hid most general suggestions in the view modal.
|
||||
assert body["suggestion_threshold_general"] == pytest.approx(0.50)
|
||||
# Default raised 0.50 → 0.70 on 2026-06-02 (alembic 0033) — 0.50
|
||||
# was too noisy in practice. The 0.70 default keeps the rail
|
||||
# signal-rich without hiding everything like the original 0.95.
|
||||
assert body["suggestion_threshold_general"] == pytest.approx(0.70)
|
||||
# Retired threshold columns must not appear in the payload.
|
||||
assert "suggestion_threshold_artist" not in body
|
||||
assert "suggestion_threshold_copyright" not in body
|
||||
|
||||
@@ -14,7 +14,7 @@ pytestmark = pytest.mark.integration
|
||||
|
||||
def test_load_or_create_writes_key_with_mode_0600(tmp_path):
|
||||
key_path = tmp_path / "secrets" / "credential_key.b64"
|
||||
CredentialCrypto(key_path)
|
||||
CredentialCrypto(key_path, bootstrap_ok=True)
|
||||
assert key_path.exists()
|
||||
mode = stat.S_IMODE(os.stat(key_path).st_mode)
|
||||
assert mode == 0o600
|
||||
@@ -25,9 +25,9 @@ def test_load_or_create_writes_key_with_mode_0600(tmp_path):
|
||||
|
||||
def test_load_existing_key_is_idempotent(tmp_path):
|
||||
key_path = tmp_path / "credential_key.b64"
|
||||
crypto1 = CredentialCrypto(key_path)
|
||||
crypto1 = CredentialCrypto(key_path, bootstrap_ok=True)
|
||||
contents_after_first = key_path.read_bytes()
|
||||
crypto2 = CredentialCrypto(key_path)
|
||||
crypto2 = CredentialCrypto(key_path, bootstrap_ok=True)
|
||||
contents_after_second = key_path.read_bytes()
|
||||
assert contents_after_first == contents_after_second
|
||||
# And both crypto instances decrypt each other's ciphertext
|
||||
@@ -36,7 +36,7 @@ def test_load_existing_key_is_idempotent(tmp_path):
|
||||
|
||||
|
||||
def test_encrypt_decrypt_round_trip(tmp_path):
|
||||
crypto = CredentialCrypto(tmp_path / "k")
|
||||
crypto = CredentialCrypto(tmp_path / "k", bootstrap_ok=True)
|
||||
plaintext = "domain.com\tTRUE\t/\tTRUE\t1700000000\tname\tvalue"
|
||||
ct = crypto.encrypt(plaintext)
|
||||
assert isinstance(ct, bytes)
|
||||
@@ -45,8 +45,27 @@ def test_encrypt_decrypt_round_trip(tmp_path):
|
||||
|
||||
|
||||
def test_decrypt_with_wrong_key_raises(tmp_path):
|
||||
crypto_a = CredentialCrypto(tmp_path / "a")
|
||||
crypto_b = CredentialCrypto(tmp_path / "b")
|
||||
crypto_a = CredentialCrypto(tmp_path / "a", bootstrap_ok=True)
|
||||
crypto_b = CredentialCrypto(tmp_path / "b", bootstrap_ok=True)
|
||||
ct = crypto_a.encrypt("secret")
|
||||
with pytest.raises(InvalidCredentialBlob):
|
||||
crypto_b.decrypt(ct)
|
||||
|
||||
|
||||
def test_missing_key_without_bootstrap_raises(tmp_path, monkeypatch):
|
||||
"""Audit 2026-06-02: without explicit opt-in, a missing key file
|
||||
is a fatal startup error — silent regeneration on partial restore
|
||||
would make every existing Credential row undecryptable."""
|
||||
from backend.app.services.credential_crypto import MissingCredentialKey
|
||||
monkeypatch.delenv("CURATOR_BOOTSTRAP_NEW_KEY", raising=False)
|
||||
with pytest.raises(MissingCredentialKey):
|
||||
CredentialCrypto(tmp_path / "absent.b64")
|
||||
|
||||
|
||||
def test_missing_key_with_env_var_bootstraps(tmp_path, monkeypatch):
|
||||
"""The env var CURATOR_BOOTSTRAP_NEW_KEY=1 is the operator's
|
||||
first-time-setup opt-in for auto-creating the key file."""
|
||||
monkeypatch.setenv("CURATOR_BOOTSTRAP_NEW_KEY", "1")
|
||||
key_path = tmp_path / "bootstrap.b64"
|
||||
CredentialCrypto(key_path) # no bootstrap_ok kwarg — relies on env
|
||||
assert key_path.exists()
|
||||
|
||||
@@ -21,7 +21,7 @@ _NETSCAPE = (
|
||||
|
||||
@pytest.fixture
|
||||
def crypto(tmp_path):
|
||||
return CredentialCrypto(tmp_path / "k")
|
||||
return CredentialCrypto(tmp_path / "k", bootstrap_ok=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -130,7 +130,7 @@ async def test_download_source_attaches_written_files(
|
||||
thumbnailer=Thumbnailer(images_root=images_root),
|
||||
settings=sync_settings,
|
||||
)
|
||||
crypto = CredentialCrypto(tmp_path / "key.b64")
|
||||
crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
|
||||
cred_service = CredentialService(db, crypto)
|
||||
|
||||
svc = DownloadService(
|
||||
@@ -405,7 +405,7 @@ async def test_backfill_decrements_after_run(
|
||||
session=db_sync, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
@@ -446,7 +446,7 @@ async def test_backfill_auto_resets_on_clean_zero_files(
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
@@ -484,7 +484,7 @@ async def test_tick_mode_does_not_touch_backfill_counter(
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
@@ -529,7 +529,7 @@ async def test_partial_error_type_maps_to_ok_status(
|
||||
session=db_sync, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
@@ -582,7 +582,7 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
|
||||
session=db_sync, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||
|
||||
# Capture the IDs that the orchestrator hands off to each Celery task.
|
||||
# The .delay() shim runs inside DownloadService._phase3_persist (lazy
|
||||
|
||||
@@ -39,8 +39,9 @@ async def test_threshold_filters_low_confidence_general(db):
|
||||
await db.flush()
|
||||
sl = await SuggestionService(db).for_image(img.id)
|
||||
names = [s.display_name for s in sl.by_category.get("general", [])]
|
||||
assert "sword" in names
|
||||
assert "lowconf" not in names
|
||||
# display_name is normalized (tag_name.normalize) before surfacing.
|
||||
assert "Sword" in names
|
||||
assert "Lowconf" not in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -84,7 +85,9 @@ async def test_raw_tag_creates_new(db):
|
||||
await db.flush()
|
||||
sl = await SuggestionService(db).for_image(img.id)
|
||||
chars = sl.by_category["character"]
|
||||
assert chars[0].display_name == "brand_new_tag"
|
||||
# display_name is the normalized Camie name (underscores -> spaces,
|
||||
# title-cased), not the raw vocab key.
|
||||
assert chars[0].display_name == "Brand New Tag"
|
||||
assert chars[0].creates_new_tag is True
|
||||
assert chars[0].canonical_tag_id is None
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import pytest
|
||||
|
||||
from backend.app.services.ml.tag_name import normalize
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw, expected",
|
||||
[
|
||||
# Rule 4: underscores -> spaces; rule 7: title case
|
||||
("light_purple_hair", "Light Purple Hair"),
|
||||
("no_pants", "No Pants"),
|
||||
("year_2005", "Year 2005"),
|
||||
# Single-word still title-cased
|
||||
("sword", "Sword"),
|
||||
# Rule 3: drop trailing _(disambiguator)
|
||||
("uchiha_sasuke_(naruto)", "Uchiha Sasuke"),
|
||||
("apple_(fruit)", "Apple"),
|
||||
("kirby_(series)", "Kirby"),
|
||||
# Repeated trailing disambig blocks
|
||||
("foo_(bar)_(baz)", "Foo"),
|
||||
# Rule 1: leading junk chars
|
||||
("#unicus_(idolmaster)", "Unicus"),
|
||||
(".52_gal_(splatoon)", "52 Gal"),
|
||||
("+_+_smile_(emote)", "Smile"),
|
||||
# Rule 5: space after colon
|
||||
("nier:automata", "Nier: Automata"),
|
||||
# Already-spaced colon left alone
|
||||
("nier: automata", "Nier: Automata"),
|
||||
# Rule 6: hyphens preserved
|
||||
("1000-nen_ikiteru_(vocaloid)", "1000-nen Ikiteru"),
|
||||
("well-known_face", "Well-known Face"),
|
||||
# Rule 2: wrapping quotes
|
||||
('"pile_em_up"_(genshin_impact)', "Pile Em Up"),
|
||||
("'foo_bar'", "Foo Bar"),
|
||||
# Rule 8: emoticons -> None
|
||||
(":/", None),
|
||||
(";)", None),
|
||||
("+_+", None),
|
||||
("^_^", None),
|
||||
# Empty / whitespace-only
|
||||
("", None),
|
||||
(" ", None),
|
||||
("___", None),
|
||||
# Apostrophe inside word — preserved, not title-cased
|
||||
("it's_okay", "It's Okay"),
|
||||
# Digit-only still surfaces (year tags)
|
||||
("2005", "2005"),
|
||||
# Multi-space collapse
|
||||
("foo___bar", "Foo Bar"),
|
||||
],
|
||||
)
|
||||
def test_normalize(raw, expected):
|
||||
assert normalize(raw) == expected
|
||||
Reference in New Issue
Block a user