Merge pull request 'Maintenance-queue health + modal/tagging keyboard pass' (#80) from dev into main
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 20s
CI / backend-lint-and-test (push) Successful in 25s
Build images / build-web (push) Successful in 2m5s
Build images / build-ml (push) Successful in 2m26s
CI / integration (push) Successful in 3m1s
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 20s
CI / backend-lint-and-test (push) Successful in 25s
Build images / build-web (push) Successful in 2m5s
Build images / build-ml (push) Successful in 2m26s
CI / integration (push) Successful in 3m1s
This commit was merged in pull request #80.
This commit is contained in:
@@ -0,0 +1,40 @@
|
|||||||
|
"""library_audit_run: resume cursor + progress timestamp for chunked scans
|
||||||
|
|
||||||
|
Revision ID: 0039
|
||||||
|
Revises: 0038
|
||||||
|
Create Date: 2026-06-07
|
||||||
|
|
||||||
|
scan_library_for_rule used to run one 2h pass that timed out on large libraries
|
||||||
|
and monopolized the concurrency-1 maintenance queue (operator-flagged). It now
|
||||||
|
runs short time-boxed chunks that re-enqueue: `resume_after_id` persists the
|
||||||
|
keyset cursor so the next chunk continues where it left off, and
|
||||||
|
`last_progress_at` lets the recovery sweep tell a progressing multi-chunk audit
|
||||||
|
from a genuinely stuck one.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0039"
|
||||||
|
down_revision: Union[str, None] = "0038"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"library_audit_run",
|
||||||
|
sa.Column(
|
||||||
|
"resume_after_id", sa.Integer, nullable=False, server_default="0"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"library_audit_run",
|
||||||
|
sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("library_audit_run", "last_progress_at")
|
||||||
|
op.drop_column("library_audit_run", "resume_after_id")
|
||||||
@@ -31,7 +31,7 @@ system_activity_bp = Blueprint(
|
|||||||
# absent.
|
# absent.
|
||||||
_QUEUE_NAMES = (
|
_QUEUE_NAMES = (
|
||||||
"default", "import", "thumbnail", "ml",
|
"default", "import", "thumbnail", "ml",
|
||||||
"download", "scan", "maintenance",
|
"download", "scan", "maintenance", "maintenance_long",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Cache module-level so all requests share the cache between polls.
|
# Cache module-level so all requests share the cache between polls.
|
||||||
|
|||||||
@@ -43,10 +43,16 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
|
"backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
|
||||||
"backend.app.tasks.download.*": {"queue": "download"},
|
"backend.app.tasks.download.*": {"queue": "download"},
|
||||||
"backend.app.tasks.scan.*": {"queue": "scan"},
|
"backend.app.tasks.scan.*": {"queue": "scan"},
|
||||||
|
# `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup
|
||||||
|
# (concurrency-1 on the scheduler). The long one-shots (DB backups,
|
||||||
|
# library audits, admin maintenance: normalize/re-extract/cascade-
|
||||||
|
# delete) run on a SEPARATE `maintenance_long` lane + worker so they
|
||||||
|
# can never starve the quick self-healing sweeps (operator-flagged
|
||||||
|
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
||||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||||
},
|
},
|
||||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||||
task_acks_late=True,
|
task_acks_late=True,
|
||||||
|
|||||||
@@ -35,3 +35,10 @@ class LibraryAuditRun(Base):
|
|||||||
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
# Chunked-scan state (alembic 0039): keyset cursor the next chunk resumes
|
||||||
|
# from, and the last time a chunk made progress (so the recovery sweep can
|
||||||
|
# tell a progressing multi-chunk audit from a stuck one).
|
||||||
|
resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
last_progress_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True,
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Tag CRUD + autocomplete + image-tag association."""
|
"""Tag CRUD + autocomplete + image-tag association."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
@@ -637,7 +638,10 @@ def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int:
|
|||||||
|
|
||||||
|
|
||||||
async def normalize_existing_tags(
|
async def normalize_existing_tags(
|
||||||
session: AsyncSession, *, dry_run: bool = False
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
dry_run: bool = False,
|
||||||
|
time_budget_seconds: float | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Convert the back-catalog to the #701 canonical tag form.
|
"""Convert the back-catalog to the #701 canonical tag form.
|
||||||
|
|
||||||
@@ -647,6 +651,14 @@ async def normalize_existing_tags(
|
|||||||
survivor to the canonical form. Idempotent: a group that is already a lone
|
survivor to the canonical form. Idempotent: a group that is already a lone
|
||||||
canonical tag is a no-op, so re-running is safe.
|
canonical tag is a no-op, so re-running is safe.
|
||||||
|
|
||||||
|
A first run over a fresh back-catalog can touch tens of thousands of tags
|
||||||
|
(the whole booru-derived vocabulary needs recasing) and won't finish inside
|
||||||
|
one Celery time limit — it timed out at 40 min (operator-flagged 2026-06-07).
|
||||||
|
`time_budget_seconds` time-boxes the live run: it stops cleanly at the budget
|
||||||
|
and reports `partial`/`remaining` so the caller can re-enqueue and continue.
|
||||||
|
Because it commits per group and is idempotent, the next run just picks up
|
||||||
|
the groups still needing change.
|
||||||
|
|
||||||
dry_run=True returns a projection (counts + a sample of the changes) with no
|
dry_run=True returns a projection (counts + a sample of the changes) with no
|
||||||
mutations. Live runs commit per group and isolate failures per group so one
|
mutations. Live runs commit per group and isolate failures per group so one
|
||||||
bad group can't strand the rest.
|
bad group can't strand the rest.
|
||||||
@@ -656,7 +668,7 @@ async def normalize_existing_tags(
|
|||||||
"total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]}
|
"total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]}
|
||||||
Returns (live):
|
Returns (live):
|
||||||
{"groups_processed", "merged", "renamed", "aliases_created", "errors",
|
{"groups_processed", "merged", "renamed", "aliases_created", "errors",
|
||||||
"sample": [...]}
|
"total_changes", "remaining", "partial", "sample": [...]}
|
||||||
"""
|
"""
|
||||||
rows = (
|
rows = (
|
||||||
await session.execute(
|
await session.execute(
|
||||||
@@ -715,9 +727,23 @@ async def normalize_existing_tags(
|
|||||||
"renamed": 0,
|
"renamed": 0,
|
||||||
"aliases_created": 0,
|
"aliases_created": 0,
|
||||||
"errors": 0,
|
"errors": 0,
|
||||||
|
"total_changes": len(touched),
|
||||||
|
"remaining": len(touched),
|
||||||
|
"partial": False,
|
||||||
"sample": sample,
|
"sample": sample,
|
||||||
}
|
}
|
||||||
for key, members in touched:
|
start = time.monotonic()
|
||||||
|
for done, (key, members) in enumerate(touched):
|
||||||
|
# Time-box: stop cleanly before the Celery limit kills us mid-group and
|
||||||
|
# strands the run as a timeout. The caller re-enqueues to finish the
|
||||||
|
# rest (idempotent — already-canonical groups are skipped next pass).
|
||||||
|
if (
|
||||||
|
time_budget_seconds is not None
|
||||||
|
and time.monotonic() - start >= time_budget_seconds
|
||||||
|
):
|
||||||
|
summary["partial"] = True
|
||||||
|
summary["remaining"] = len(touched) - done
|
||||||
|
break
|
||||||
canonical = key[2]
|
canonical = key[2]
|
||||||
names_by_id = dict(members)
|
names_by_id = dict(members)
|
||||||
# Survivor: prefer a member already named canonically (no rename, no
|
# Survivor: prefer a member already named canonically (no rename, no
|
||||||
@@ -752,4 +778,7 @@ async def normalize_existing_tags(
|
|||||||
log.warning(
|
log.warning(
|
||||||
"tag normalize failed for group %r: %s", canonical, exc
|
"tag normalize failed for group %r: %s", canonical, exc
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
# Loop finished without hitting the time budget — nothing left to do.
|
||||||
|
summary["remaining"] = 0
|
||||||
return summary
|
return summary
|
||||||
|
|||||||
@@ -75,6 +75,15 @@ def reextract_archive_attachments_task(self) -> dict:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Time-box one chunk well under the soft limit so a large back-catalog (the
|
||||||
|
# first run recases the whole booru vocabulary) can't run the task into the
|
||||||
|
# Celery time limit — it timed out at 40 min, operator-flagged 2026-06-07. The
|
||||||
|
# task re-enqueues itself until nothing remains (idempotent — already-canonical
|
||||||
|
# groups are skipped). 600s keeps each chunk short enough that the recovery
|
||||||
|
# sweep and other maintenance tasks interleave on the concurrency-1 queue.
|
||||||
|
_NORMALIZE_CHUNK_SECONDS = 600
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
@celery.task(
|
||||||
name="backend.app.tasks.admin.normalize_tags_task",
|
name="backend.app.tasks.admin.normalize_tags_task",
|
||||||
bind=True,
|
bind=True,
|
||||||
@@ -85,8 +94,9 @@ def reextract_archive_attachments_task(self) -> dict:
|
|||||||
def normalize_tags_task(self) -> dict:
|
def normalize_tags_task(self) -> dict:
|
||||||
"""Wraps tag_service.normalize_existing_tags (#714): Title-Case the
|
"""Wraps tag_service.normalize_existing_tags (#714): Title-Case the
|
||||||
back-catalog and merge case/whitespace-variant duplicate tags via the
|
back-catalog and merge case/whitespace-variant duplicate tags via the
|
||||||
tested async merge path. Runs under its own asyncio loop + per-task async
|
tested async merge path. Time-boxed + self-resuming so a huge first run
|
||||||
engine (NullPool, disposed when the loop ends), mirroring download_source."""
|
finishes across chunks instead of timing out. Runs under its own asyncio
|
||||||
|
loop + per-task async engine (NullPool), mirroring download_source."""
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from ..services.tag_service import normalize_existing_tags
|
from ..services.tag_service import normalize_existing_tags
|
||||||
@@ -97,8 +107,20 @@ def normalize_tags_task(self) -> dict:
|
|||||||
try:
|
try:
|
||||||
async with async_factory() as session:
|
async with async_factory() as session:
|
||||||
# normalize_existing_tags commits per group internally.
|
# normalize_existing_tags commits per group internally.
|
||||||
return await normalize_existing_tags(session, dry_run=False)
|
return await normalize_existing_tags(
|
||||||
|
session, dry_run=False,
|
||||||
|
time_budget_seconds=_NORMALIZE_CHUNK_SECONDS,
|
||||||
|
)
|
||||||
finally:
|
finally:
|
||||||
await async_engine.dispose()
|
await async_engine.dispose()
|
||||||
|
|
||||||
return asyncio.run(_run())
|
summary = asyncio.run(_run())
|
||||||
|
# More groups to canonicalize than fit this chunk — continue in the next.
|
||||||
|
if summary.get("partial") and summary.get("remaining", 0) > 0:
|
||||||
|
log.info(
|
||||||
|
"normalize_tags_task chunk done (%d processed, %d remaining) — "
|
||||||
|
"re-enqueuing to continue",
|
||||||
|
summary.get("groups_processed", 0), summary["remaining"],
|
||||||
|
)
|
||||||
|
normalize_tags_task.delay()
|
||||||
|
return summary
|
||||||
|
|||||||
@@ -41,7 +41,11 @@ def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
|
|||||||
bind=True,
|
bind=True,
|
||||||
autoretry_for=(OperationalError, DBAPIError),
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
retry_backoff=10, retry_backoff_max=120, max_retries=2,
|
retry_backoff=10, retry_backoff_max=120, max_retries=2,
|
||||||
soft_time_limit=600, time_limit=720,
|
# A pg_dump can't be chunked; the 12-min limit timed out once the DB grew
|
||||||
|
# (operator-flagged 2026-06-07). 30/35 min gives real headroom. (A long
|
||||||
|
# backup still briefly holds the concurrency-1 maintenance lane — the
|
||||||
|
# structural fix is a dedicated lane for the long one-shots.)
|
||||||
|
soft_time_limit=1800, time_limit=2100,
|
||||||
)
|
)
|
||||||
def backup_db_task(self, *, tag: str | None = None,
|
def backup_db_task(self, *, tag: str | None = None,
|
||||||
triggered_by: str = "manual") -> dict:
|
triggered_by: str = "manual") -> dict:
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ State machine:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
@@ -31,6 +32,12 @@ log = logging.getLogger(__name__)
|
|||||||
_BATCH = 500
|
_BATCH = 500
|
||||||
_PROGRESS_TICK = 100
|
_PROGRESS_TICK = 100
|
||||||
_MAX_MATCHED = 50_000
|
_MAX_MATCHED = 50_000
|
||||||
|
# One chunk's wall-clock budget. Was a single 2h pass that timed out on large
|
||||||
|
# libraries and held the concurrency-1 maintenance queue the whole time
|
||||||
|
# (operator-flagged 2026-06-07). Now: scan ~10 min, persist the keyset cursor +
|
||||||
|
# matches, re-enqueue to continue — so backups/vacuum/normalize chunks can
|
||||||
|
# interleave. soft/hard limits sit just above so the budget fires first.
|
||||||
|
_CHUNK_SECONDS = 600
|
||||||
|
|
||||||
_RULES = {
|
_RULES = {
|
||||||
"transparency": transparency.evaluate,
|
"transparency": transparency.evaluate,
|
||||||
@@ -46,13 +53,16 @@ _RULES = {
|
|||||||
retry_backoff_max=60,
|
retry_backoff_max=60,
|
||||||
retry_jitter=True,
|
retry_jitter=True,
|
||||||
max_retries=3,
|
max_retries=3,
|
||||||
soft_time_limit=7200,
|
soft_time_limit=900,
|
||||||
time_limit=7500,
|
time_limit=1000,
|
||||||
)
|
)
|
||||||
def scan_library_for_rule(self, audit_id: int) -> dict:
|
def scan_library_for_rule(self, audit_id: int) -> dict:
|
||||||
"""See module docstring. Returns a small summary dict for eager-mode
|
"""See module docstring. Time-boxed + self-resuming: one call scans a
|
||||||
|
~10-min chunk, persists the resume cursor + matches, and re-enqueues itself
|
||||||
|
until the library is exhausted. Returns a small summary dict for eager-mode
|
||||||
test assertions (real workers ignore the return value)."""
|
test assertions (real workers ignore the return value)."""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
|
start = time.monotonic()
|
||||||
try:
|
try:
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
audit = session.get(LibraryAuditRun, audit_id)
|
audit = session.get(LibraryAuditRun, audit_id)
|
||||||
@@ -63,9 +73,10 @@ def scan_library_for_rule(self, audit_id: int) -> dict:
|
|||||||
_mark_error(session, audit_id, f"unknown rule {audit.rule!r}")
|
_mark_error(session, audit_id, f"unknown rule {audit.rule!r}")
|
||||||
return {"audit_id": audit_id, "status": "error"}
|
return {"audit_id": audit_id, "status": "error"}
|
||||||
params = dict(audit.params or {})
|
params = dict(audit.params or {})
|
||||||
matched: list[int] = []
|
# Resume from the previous chunk's persisted state.
|
||||||
scanned = 0
|
matched: list[int] = list(audit.matched_ids or [])
|
||||||
last_id = 0
|
scanned = audit.scanned_count or 0
|
||||||
|
last_id = audit.resume_after_id or 0
|
||||||
while True:
|
while True:
|
||||||
# Cancellation check between batches.
|
# Cancellation check between batches.
|
||||||
current_status = session.execute(
|
current_status = session.execute(
|
||||||
@@ -74,6 +85,15 @@ def scan_library_for_rule(self, audit_id: int) -> dict:
|
|||||||
).scalar_one()
|
).scalar_one()
|
||||||
if current_status == "cancelled":
|
if current_status == "cancelled":
|
||||||
return {"audit_id": audit_id, "status": "cancelled"}
|
return {"audit_id": audit_id, "status": "cancelled"}
|
||||||
|
# Time-box: persist the cursor + matches and re-enqueue so the
|
||||||
|
# queue is freed between chunks. The next call resumes here.
|
||||||
|
if time.monotonic() - start >= _CHUNK_SECONDS:
|
||||||
|
_persist_chunk(session, audit_id, scanned, matched, last_id)
|
||||||
|
scan_library_for_rule.delay(audit_id)
|
||||||
|
return {
|
||||||
|
"audit_id": audit_id, "status": "running",
|
||||||
|
"partial": True, "scanned": scanned,
|
||||||
|
}
|
||||||
rows = session.execute(
|
rows = session.execute(
|
||||||
select(ImageRecord.id, ImageRecord.path)
|
select(ImageRecord.id, ImageRecord.path)
|
||||||
.where(ImageRecord.id > last_id)
|
.where(ImageRecord.id > last_id)
|
||||||
@@ -114,10 +134,16 @@ def scan_library_for_rule(self, audit_id: int) -> dict:
|
|||||||
)
|
)
|
||||||
return {"audit_id": audit_id, "status": "error"}
|
return {"audit_id": audit_id, "status": "error"}
|
||||||
if scanned % _PROGRESS_TICK == 0:
|
if scanned % _PROGRESS_TICK == 0:
|
||||||
|
# Cheap heartbeat: scanned_count + last_progress_at so the
|
||||||
|
# recovery sweep sees the multi-chunk audit is alive. The
|
||||||
|
# cursor + matches are persisted at chunk boundaries.
|
||||||
session.execute(
|
session.execute(
|
||||||
update(LibraryAuditRun)
|
update(LibraryAuditRun)
|
||||||
.where(LibraryAuditRun.id == audit_id)
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
.values(scanned_count=scanned)
|
.values(
|
||||||
|
scanned_count=scanned,
|
||||||
|
last_progress_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
# Final state.
|
# Final state.
|
||||||
@@ -128,8 +154,10 @@ def scan_library_for_rule(self, audit_id: int) -> dict:
|
|||||||
scanned_count=scanned,
|
scanned_count=scanned,
|
||||||
matched_count=len(matched),
|
matched_count=len(matched),
|
||||||
matched_ids=matched,
|
matched_ids=matched,
|
||||||
|
resume_after_id=last_id,
|
||||||
status="ready",
|
status="ready",
|
||||||
finished_at=datetime.now(UTC),
|
finished_at=datetime.now(UTC),
|
||||||
|
last_progress_at=datetime.now(UTC),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
@@ -140,9 +168,14 @@ def scan_library_for_rule(self, audit_id: int) -> dict:
|
|||||||
"matched": len(matched),
|
"matched": len(matched),
|
||||||
}
|
}
|
||||||
except SoftTimeLimitExceeded:
|
except SoftTimeLimitExceeded:
|
||||||
with SessionLocal() as session:
|
# Backstop (the in-chunk budget should fire first): the audit stays
|
||||||
_mark_error(session, audit_id, "soft_time_limit exceeded (>7200s)")
|
# 'running' with its last committed cursor; re-enqueue to continue from
|
||||||
raise
|
# there rather than marking the whole run an error.
|
||||||
|
log.warning(
|
||||||
|
"audit %s: soft time limit hit — re-enqueuing to resume", audit_id,
|
||||||
|
)
|
||||||
|
scan_library_for_rule.delay(audit_id)
|
||||||
|
return {"audit_id": audit_id, "status": "running", "partial": True}
|
||||||
except (OperationalError, DBAPIError):
|
except (OperationalError, DBAPIError):
|
||||||
# Retryable per the decorator; leave row in 'running' and let
|
# Retryable per the decorator; leave row in 'running' and let
|
||||||
# autoretry try again. Recovery sweep catches if all retries fail.
|
# autoretry try again. Recovery sweep catches if all retries fail.
|
||||||
@@ -154,6 +187,23 @@ def scan_library_for_rule(self, audit_id: int) -> dict:
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _persist_chunk(session, audit_id, scanned, matched, last_id) -> None:
|
||||||
|
"""Persist a chunk boundary: scanned count, matches so far, and the keyset
|
||||||
|
cursor the next chunk resumes from. Keeps status='running'."""
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(
|
||||||
|
scanned_count=scanned,
|
||||||
|
matched_count=len(matched),
|
||||||
|
matched_ids=list(matched),
|
||||||
|
resume_after_id=last_id,
|
||||||
|
last_progress_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
def _mark_error(session, audit_id: int, error_msg: str) -> None:
|
def _mark_error(session, audit_id: int, error_msg: str) -> None:
|
||||||
session.execute(
|
session.execute(
|
||||||
update(LibraryAuditRun)
|
update(LibraryAuditRun)
|
||||||
|
|||||||
@@ -647,19 +647,30 @@ def recover_stalled_library_audit_runs() -> int:
|
|||||||
guard in start_audit_run — a SIGKILL'd run would block all future
|
guard in start_audit_run — a SIGKILL'd run would block all future
|
||||||
audits until manual DB surgery. (The guard is now age-aware, but
|
audits until manual DB surgery. (The guard is now age-aware, but
|
||||||
this sweep is what makes that work in practice.)
|
this sweep is what makes that work in practice.)
|
||||||
|
|
||||||
|
Measures staleness from last_progress_at (alembic 0039), NOT started_at:
|
||||||
|
a chunked scan stays 'running' across many re-enqueued chunks and can
|
||||||
|
legitimately run for hours on a big library — only flag one that hasn't
|
||||||
|
made progress in the threshold window (a dead chunk that never re-enqueued).
|
||||||
|
Falls back to started_at for pre-0039 / never-ticked rows.
|
||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
cutoff = now - timedelta(minutes=LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES)
|
cutoff = now - timedelta(minutes=LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES)
|
||||||
msg = (
|
msg = (
|
||||||
f"stranded by recovery sweep (no terminal status after "
|
f"stranded by recovery sweep (no progress for "
|
||||||
f"{LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES} min)"
|
f"{LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES} min)"
|
||||||
)
|
)
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
result = session.execute(
|
result = session.execute(
|
||||||
update(LibraryAuditRun)
|
update(LibraryAuditRun)
|
||||||
.where(LibraryAuditRun.status == "running")
|
.where(LibraryAuditRun.status == "running")
|
||||||
.where(LibraryAuditRun.started_at < cutoff)
|
.where(
|
||||||
|
func.coalesce(
|
||||||
|
LibraryAuditRun.last_progress_at,
|
||||||
|
LibraryAuditRun.started_at,
|
||||||
|
) < cutoff
|
||||||
|
)
|
||||||
.values(status="error", finished_at=now, error=msg)
|
.values(status="error", finished_at=now, error=msg)
|
||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|||||||
@@ -35,6 +35,15 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./backend:/app/backend
|
- ./backend:/app/backend
|
||||||
|
|
||||||
|
maintenance-long:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
environment:
|
||||||
|
LOG_LEVEL: DEBUG
|
||||||
|
volumes:
|
||||||
|
- ./backend:/app/backend
|
||||||
|
|
||||||
ml-worker:
|
ml-worker:
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
|
|||||||
+26
-3
@@ -21,6 +21,11 @@ services:
|
|||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
image: pgvector/pgvector:pg16
|
image: pgvector/pgvector:pg16
|
||||||
|
# Docker's default /dev/shm is 64MB; VACUUM (ANALYZE) and parallel queries
|
||||||
|
# allocate larger shared-memory segments and fail with
|
||||||
|
# "could not resize shared memory segment ... No space left on device"
|
||||||
|
# (operator-flagged 2026-06-07, vacuum_analyze on import_task needed 67MB).
|
||||||
|
shm_size: 512m
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: ${DB_USER:-fabledcurator}
|
POSTGRES_USER: ${DB_USER:-fabledcurator}
|
||||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-fabledcurator_dev}
|
POSTGRES_PASSWORD: ${DB_PASSWORD:-fabledcurator_dev}
|
||||||
@@ -52,7 +57,6 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./images:/images
|
- ./images:/images
|
||||||
- ./import:/import
|
- ./import:/import
|
||||||
- ./downloads:/downloads
|
|
||||||
# FC-5 legacy migration: bind-mount the host's ImageRepo images dir
|
# FC-5 legacy migration: bind-mount the host's ImageRepo images dir
|
||||||
# under /import (FC's existing filesystem scan picks them up). Read-only
|
# under /import (FC's existing filesystem scan picks them up). Read-only
|
||||||
# is sufficient — FC copies into /images during the scan. The worker +
|
# is sufficient — FC copies into /images during the scan. The worker +
|
||||||
@@ -71,10 +75,11 @@ services:
|
|||||||
<<: *app_env
|
<<: *app_env
|
||||||
CELERY_QUEUES: default,import,thumbnail,download
|
CELERY_QUEUES: default,import,thumbnail,download
|
||||||
CELERY_CONCURRENCY: "2"
|
CELERY_CONCURRENCY: "2"
|
||||||
|
# /downloads dropped — nothing in the app references it (operator-flagged
|
||||||
|
# 2026-06-07: it wasn't mapped in prod and everything worked).
|
||||||
volumes:
|
volumes:
|
||||||
- ./images:/images
|
- ./images:/images
|
||||||
- ./import:/import
|
- ./import:/import
|
||||||
- ./downloads:/downloads
|
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres: { condition: service_healthy }
|
postgres: { condition: service_healthy }
|
||||||
redis: { condition: service_healthy }
|
redis: { condition: service_healthy }
|
||||||
@@ -88,7 +93,25 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./images:/images
|
- ./images:/images
|
||||||
- ./import:/import
|
- ./import:/import
|
||||||
- ./downloads:/downloads
|
depends_on:
|
||||||
|
postgres: { condition: service_healthy }
|
||||||
|
redis: { condition: service_healthy }
|
||||||
|
|
||||||
|
# Dedicated lane for long one-shot maintenance (DB backups, library audits,
|
||||||
|
# admin maintenance). Kept off the scheduler's quick `maintenance` lane so a
|
||||||
|
# 30-min backup or a multi-chunk audit can never starve the 5-min recovery
|
||||||
|
# sweeps / vacuum (operator-flagged 2026-06-07). One slot — these are heavy.
|
||||||
|
maintenance-long:
|
||||||
|
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||||
|
command: ["worker"]
|
||||||
|
environment:
|
||||||
|
<<: *app_env
|
||||||
|
CELERY_QUEUES: maintenance_long
|
||||||
|
CELERY_CONCURRENCY: "1"
|
||||||
|
# Only /images: backups write to /images/_backups, audits read /images, and
|
||||||
|
# the admin tasks (re-extract/cascade-delete/normalize) operate on /images.
|
||||||
|
volumes:
|
||||||
|
- ./images:/images
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres: { condition: service_healthy }
|
postgres: { condition: service_healthy }
|
||||||
redis: { condition: service_healthy }
|
redis: { condition: service_healthy }
|
||||||
|
|||||||
@@ -2,12 +2,17 @@
|
|||||||
<v-card>
|
<v-card>
|
||||||
<v-card-title>Pick a fandom</v-card-title>
|
<v-card-title>Pick a fandom</v-card-title>
|
||||||
<v-card-text>
|
<v-card-text>
|
||||||
|
<!-- A2/A3 (2026-06-07): autofocus so the operator types immediately, and
|
||||||
|
picking a fandom (keyboard: type → arrow → Enter, or a click) confirms
|
||||||
|
in one step — selecting IS the decision in this dialog. -->
|
||||||
<v-autocomplete
|
<v-autocomplete
|
||||||
v-model="selectedId"
|
v-model="selectedId"
|
||||||
:items="store.fandomCache"
|
:items="store.fandomCache"
|
||||||
:item-title="(f) => f.name"
|
:item-title="(f) => f.name"
|
||||||
:item-value="(f) => f.id"
|
:item-value="(f) => f.id"
|
||||||
label="Fandom" clearable density="compact"
|
label="Fandom" clearable density="compact"
|
||||||
|
autofocus
|
||||||
|
@update:model-value="onSelect"
|
||||||
/>
|
/>
|
||||||
<v-divider class="my-3" />
|
<v-divider class="my-3" />
|
||||||
<p class="text-caption mb-2">Or create a new fandom:</p>
|
<p class="text-caption mb-2">Or create a new fandom:</p>
|
||||||
@@ -50,6 +55,12 @@ function onConfirm() {
|
|||||||
const f = store.fandomCache.find(x => x.id === selectedId.value)
|
const f = store.fandomCache.find(x => x.id === selectedId.value)
|
||||||
if (f) emit('confirm', f)
|
if (f) emit('confirm', f)
|
||||||
}
|
}
|
||||||
|
// Picking a fandom (keyboard Enter or click) confirms immediately. Ignore the
|
||||||
|
// clear action (null) so clearing the field doesn't fire a confirm.
|
||||||
|
function onSelect(id) {
|
||||||
|
if (id == null) return
|
||||||
|
onConfirm()
|
||||||
|
}
|
||||||
// Create the character with no fandom. Emits null so the caller knows this
|
// Create the character with no fandom. Emits null so the caller knows this
|
||||||
// was a deliberate "unassigned", not a cancel.
|
// was a deliberate "unassigned", not a cancel.
|
||||||
function onNoFandom() {
|
function onNoFandom() {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@
|
|||||||
:items="store.fandomCache"
|
:items="store.fandomCache"
|
||||||
:item-title="(f) => f.name" :item-value="(f) => f.id"
|
:item-title="(f) => f.name" :item-value="(f) => f.id"
|
||||||
label="Fandom" clearable density="compact"
|
label="Fandom" clearable density="compact"
|
||||||
|
autofocus
|
||||||
:hint="selectedId == null
|
:hint="selectedId == null
|
||||||
? 'No fandom — the character will be unassigned.' : ''"
|
? 'No fandom — the character will be unassigned.' : ''"
|
||||||
persistent-hint
|
persistent-hint
|
||||||
|
|||||||
@@ -8,6 +8,25 @@
|
|||||||
<v-icon>mdi-close</v-icon>
|
<v-icon>mdi-close</v-icon>
|
||||||
</button>
|
</button>
|
||||||
|
|
||||||
|
<!-- C8: keyboard cheatsheet. '?' toggles; the corner hint advertises it. -->
|
||||||
|
<button
|
||||||
|
class="fc-viewer__help-hint" aria-label="Keyboard shortcuts (press ?)"
|
||||||
|
@click="showHelp = !showHelp"
|
||||||
|
>?</button>
|
||||||
|
<div v-if="showHelp" class="fc-viewer__help" @click.self="showHelp = false">
|
||||||
|
<div class="fc-viewer__help-card" role="dialog" aria-label="Keyboard shortcuts">
|
||||||
|
<h3>Keyboard shortcuts</h3>
|
||||||
|
<dl>
|
||||||
|
<div><dt>← / →</dt><dd>Previous / next image</dd></div>
|
||||||
|
<div><dt>T or /</dt><dd>Jump to the tag input</dd></div>
|
||||||
|
<div><dt>↑ / ↓</dt><dd>Move through tag suggestions</dd></div>
|
||||||
|
<div><dt>Enter / Tab</dt><dd>Accept the highlighted tag</dd></div>
|
||||||
|
<div><dt>Esc</dt><dd>Close a dialog, then the viewer</dd></div>
|
||||||
|
<div><dt>?</dt><dd>Toggle this help</dd></div>
|
||||||
|
</dl>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<v-chip
|
<v-chip
|
||||||
v-if="integrityBadge"
|
v-if="integrityBadge"
|
||||||
:color="integrityBadge.color"
|
:color="integrityBadge.color"
|
||||||
@@ -63,7 +82,7 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||||
import { useModalStore } from '../../stores/modal.js'
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import { arrowNavAllowed } from '../../utils/textEntry.js'
|
import { arrowNavAllowed, isTextEntry } from '../../utils/textEntry.js'
|
||||||
import ImageCanvas from './ImageCanvas.vue'
|
import ImageCanvas from './ImageCanvas.vue'
|
||||||
import VideoCanvas from './VideoCanvas.vue'
|
import VideoCanvas from './VideoCanvas.vue'
|
||||||
import TagPanel from './TagPanel.vue'
|
import TagPanel from './TagPanel.vue'
|
||||||
@@ -74,6 +93,7 @@ const emit = defineEmits(['close'])
|
|||||||
|
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const rootEl = ref(null)
|
const rootEl = ref(null)
|
||||||
|
const showHelp = ref(false)
|
||||||
|
|
||||||
const isVideo = computed(() =>
|
const isVideo = computed(() =>
|
||||||
modal.current?.mime && modal.current.mime.startsWith('video/')
|
modal.current?.mime && modal.current.mime.startsWith('video/')
|
||||||
@@ -96,21 +116,27 @@ let prevBodyOverflow = null
|
|||||||
// that. Filtered via isTextEntry so tag/comment inputs still get their
|
// that. Filtered via isTextEntry so tag/comment inputs still get their
|
||||||
// own keystrokes.
|
// own keystrokes.
|
||||||
function onKeyDown(ev) {
|
function onKeyDown(ev) {
|
||||||
|
if (ev.key === 'Escape' && showHelp.value) {
|
||||||
|
// Close the cheatsheet first; a second Esc closes the modal.
|
||||||
|
ev.preventDefault()
|
||||||
|
showHelp.value = false
|
||||||
|
return
|
||||||
|
}
|
||||||
if (ev.key === 'Escape') {
|
if (ev.key === 'Escape') {
|
||||||
// Escape closes the modal even from inside a text input — that's
|
// Escape closes the modal even from inside a text input — the universal
|
||||||
// the universal "get me out of here" expectation, and the
|
// "get me out of here" expectation; the autofocused tag field would
|
||||||
// autofocused tag-entry field would otherwise trap focus with no
|
// otherwise trap focus (operator-flagged 2026-06-01). EXCEPTION: when the
|
||||||
// visible escape (operator-flagged 2026-06-01). EXCEPTION: when a
|
// keystroke originates INSIDE an open Vuetify overlay's content (a rename/
|
||||||
// nested Vuetify overlay is open (v-menu autocomplete dropdown,
|
// fandom/alias dialog, or a kebab menu), let that overlay handle its own
|
||||||
// FandomPicker v-dialog, per-suggestion 3-dot menu), let that
|
// Esc and don't close the whole modal.
|
||||||
// overlay's own Esc handling fire instead of closing the whole
|
//
|
||||||
// modal mid-interaction. Vuetify marks open overlays with
|
// #700 re-fix (2026-06-07): the prior guard queried for ANY active overlay
|
||||||
// `.v-overlay--active`.
|
// anywhere in the DOM and suppressed the close — so a lingering overlay
|
||||||
// EXCLUDE tooltips (`.v-tooltip`): they're also `.v-overlay--active` while
|
// after accepting a suggestion (focus drops to <body>, not into any
|
||||||
// shown, so one lingering after a hover/click (e.g. just after accepting a
|
// overlay) wrongly blocked Esc. Keying off the event's origin instead means
|
||||||
// suggested tag) wrongly suppressed the close (#700). Only real interactive
|
// a stray overlay no longer traps the modal: only an Esc pressed from
|
||||||
// overlays (menus/dialogs) should keep ESC from closing the modal.
|
// within overlay content defers to that overlay.
|
||||||
if (document.querySelector('.v-overlay--active:not(.v-tooltip)')) return
|
if (ev.target?.closest?.('.v-overlay__content')) return
|
||||||
ev.preventDefault()
|
ev.preventDefault()
|
||||||
emit('close')
|
emit('close')
|
||||||
} else if (ev.key === 'ArrowLeft') {
|
} else if (ev.key === 'ArrowLeft') {
|
||||||
@@ -123,6 +149,14 @@ function onKeyDown(ev) {
|
|||||||
if (!arrowNavAllowed(ev.target)) return
|
if (!arrowNavAllowed(ev.target)) return
|
||||||
ev.preventDefault()
|
ev.preventDefault()
|
||||||
modal.goNext()
|
modal.goNext()
|
||||||
|
} else if ((ev.key === '/' || ev.key === 't') && !isTextEntry(ev.target)) {
|
||||||
|
// C9 (2026-06-07): jump focus to the tag input from anywhere in the modal.
|
||||||
|
const input = document.querySelector('.fc-tag-autocomplete input')
|
||||||
|
if (input) { ev.preventDefault(); input.focus() }
|
||||||
|
} else if (ev.key === '?' && !isTextEntry(ev.target)) {
|
||||||
|
// C8: toggle the keyboard cheatsheet.
|
||||||
|
ev.preventDefault()
|
||||||
|
showHelp.value = !showHelp.value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,6 +212,43 @@ function nextFrame() {
|
|||||||
}
|
}
|
||||||
.fc-viewer__nav:disabled { opacity: 0.3; cursor: not-allowed; }
|
.fc-viewer__nav:disabled { opacity: 0.3; cursor: not-allowed; }
|
||||||
.fc-viewer__close { top: 16px; right: 16px; transform: none; }
|
.fc-viewer__close { top: 16px; right: 16px; transform: none; }
|
||||||
|
.fc-viewer__help-hint {
|
||||||
|
position: absolute; top: 16px; right: 64px; z-index: 2;
|
||||||
|
width: 32px; height: 32px; border-radius: 50%;
|
||||||
|
background: rgba(255, 255, 255, 0.08);
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
font-weight: 700; cursor: pointer; opacity: 0.6;
|
||||||
|
}
|
||||||
|
.fc-viewer__help-hint:hover { opacity: 1; }
|
||||||
|
.fc-viewer__help {
|
||||||
|
position: absolute; inset: 0; z-index: 4;
|
||||||
|
display: flex; align-items: center; justify-content: center;
|
||||||
|
background: rgba(0, 0, 0, 0.4);
|
||||||
|
}
|
||||||
|
.fc-viewer__help-card {
|
||||||
|
background: rgb(var(--v-theme-surface));
|
||||||
|
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||||
|
border-radius: 10px; padding: 20px 24px; min-width: 320px;
|
||||||
|
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.5);
|
||||||
|
}
|
||||||
|
.fc-viewer__help-card h3 {
|
||||||
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-size: 18px; margin-bottom: 12px;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
|
.fc-viewer__help-card dl { display: flex; flex-direction: column; gap: 8px; }
|
||||||
|
.fc-viewer__help-card dl > div {
|
||||||
|
display: flex; gap: 16px; align-items: baseline;
|
||||||
|
}
|
||||||
|
.fc-viewer__help-card dt {
|
||||||
|
flex: 0 0 96px; text-align: right;
|
||||||
|
font-family: 'JetBrains Mono', monospace; font-size: 12px;
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
}
|
||||||
|
.fc-viewer__help-card dd {
|
||||||
|
flex: 1; font-size: 14px;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
.fc-viewer__integrity {
|
.fc-viewer__integrity {
|
||||||
position: absolute; top: 72px; right: 16px; z-index: 3;
|
position: absolute; top: 72px; right: 16px; z-index: 3;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,10 +8,12 @@
|
|||||||
@keydown.down.prevent="moveHighlight(1)"
|
@keydown.down.prevent="moveHighlight(1)"
|
||||||
@keydown.up.prevent="moveHighlight(-1)"
|
@keydown.up.prevent="moveHighlight(-1)"
|
||||||
@keydown.enter.prevent="onEnter"
|
@keydown.enter.prevent="onEnter"
|
||||||
|
@keydown.tab="onTab"
|
||||||
@keydown.esc="$emit('cancel')"
|
@keydown.esc="$emit('cancel')"
|
||||||
/>
|
/>
|
||||||
<v-list
|
<v-list
|
||||||
v-if="hits.length || allowCreate"
|
v-if="hits.length || allowCreate"
|
||||||
|
ref="listRef"
|
||||||
density="compact" class="fc-tag-autocomplete__list"
|
density="compact" class="fc-tag-autocomplete__list"
|
||||||
>
|
>
|
||||||
<v-list-item
|
<v-list-item
|
||||||
@@ -85,6 +87,7 @@ onMounted(() => {
|
|||||||
const query = ref('')
|
const query = ref('')
|
||||||
const hits = ref([])
|
const hits = ref([])
|
||||||
const highlight = ref(0)
|
const highlight = ref(0)
|
||||||
|
const listRef = ref(null)
|
||||||
const fandomDialog = ref(false)
|
const fandomDialog = ref(false)
|
||||||
let pendingNewName = null
|
let pendingNewName = null
|
||||||
|
|
||||||
@@ -137,6 +140,14 @@ function moveHighlight (delta) {
|
|||||||
const total = hits.value.length + (allowCreate.value ? 1 : 0)
|
const total = hits.value.length + (allowCreate.value ? 1 : 0)
|
||||||
if (total === 0) return
|
if (total === 0) return
|
||||||
highlight.value = (highlight.value + delta + total) % total
|
highlight.value = (highlight.value + delta + total) % total
|
||||||
|
// Keep the highlighted row visible — the list is capped at 240px and arrowing
|
||||||
|
// past the fold otherwise left the active item off-screen (operator-flagged
|
||||||
|
// 2026-06-07). block:'nearest' scrolls the minimum needed.
|
||||||
|
nextTick(() => {
|
||||||
|
listRef.value?.$el
|
||||||
|
?.querySelector('.v-list-item--active')
|
||||||
|
?.scrollIntoView({ block: 'nearest' })
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function onPick (hit) { emit('pick-existing', hit); reset() }
|
function onPick (hit) { emit('pick-existing', hit); reset() }
|
||||||
@@ -177,6 +188,16 @@ function onEnter () {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// B5 (2026-06-07): when the suggestion list is open, Tab accepts the
|
||||||
|
// highlighted row (standard autocomplete convention) instead of leaving the
|
||||||
|
// field. With the list closed it falls through to normal focus traversal.
|
||||||
|
function onTab (e) {
|
||||||
|
if (hits.value.length || allowCreate.value) {
|
||||||
|
e.preventDefault()
|
||||||
|
onEnter()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function reset () { query.value = ''; hits.value = []; highlight.value = 0 }
|
function reset () { query.value = ''; hits.value = []; highlight.value = 0 }
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
<template>
|
||||||
|
<!-- One tag chip + its kebab. The kebab uses the SAME explicit-menu pattern
|
||||||
|
as SuggestionItem (activator="parent" + :open-on-click="false" + a manual
|
||||||
|
v-model), because the #activator / v-bind="props" pattern never toggles a
|
||||||
|
v-menu inside the teleported ImageViewer modal (#711, re-fixed 2026-06-07
|
||||||
|
after the first attempt used the broken pattern). -->
|
||||||
|
<span class="fc-tag-chip">
|
||||||
|
<v-chip
|
||||||
|
size="small" closable
|
||||||
|
:color="store.colorFor(tag.kind)" variant="tonal"
|
||||||
|
@click:close="$emit('remove', tag.id)"
|
||||||
|
>
|
||||||
|
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
||||||
|
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
||||||
|
</v-chip>
|
||||||
|
<span class="fc-tag-chip__menu-wrap">
|
||||||
|
<v-btn
|
||||||
|
class="fc-tag-chip__kebab"
|
||||||
|
icon="mdi-dots-vertical" size="x-small"
|
||||||
|
variant="text" density="comfortable"
|
||||||
|
:aria-label="`More actions for ${tag.name}`"
|
||||||
|
@click.stop="menuOpen = !menuOpen"
|
||||||
|
/>
|
||||||
|
<v-menu v-model="menuOpen" activator="parent" :open-on-click="false">
|
||||||
|
<v-list density="compact">
|
||||||
|
<v-list-item @click="$emit('rename', tag)">
|
||||||
|
<v-list-item-title>Rename…</v-list-item-title>
|
||||||
|
</v-list-item>
|
||||||
|
<v-list-item v-if="tag.kind === 'character'" @click="$emit('set-fandom', tag)">
|
||||||
|
<v-list-item-title>Set fandom…</v-list-item-title>
|
||||||
|
</v-list-item>
|
||||||
|
</v-list>
|
||||||
|
</v-menu>
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref } from 'vue'
|
||||||
|
import { useTagStore } from '../../stores/tags.js'
|
||||||
|
|
||||||
|
defineProps({ tag: { type: Object, required: true } })
|
||||||
|
defineEmits(['remove', 'rename', 'set-fandom'])
|
||||||
|
|
||||||
|
const store = useTagStore()
|
||||||
|
const menuOpen = ref(false)
|
||||||
|
|
||||||
|
const KIND_ICONS = {
|
||||||
|
general: 'mdi-tag', character: 'mdi-account-circle',
|
||||||
|
fandom: 'mdi-book-open-page-variant', series: 'mdi-bookshelf',
|
||||||
|
meta: 'mdi-cog-outline', rating: 'mdi-shield-check-outline',
|
||||||
|
}
|
||||||
|
function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-tag-chip { display: inline-flex; align-items: center; gap: 1px; }
|
||||||
|
.fc-tag-chip__menu-wrap { display: inline-flex; align-items: center; }
|
||||||
|
.fc-tag-chip__kebab { opacity: 0.7; }
|
||||||
|
.fc-tag-chip:hover .fc-tag-chip__kebab { opacity: 1; }
|
||||||
|
</style>
|
||||||
@@ -2,43 +2,11 @@
|
|||||||
<aside class="fc-tag-panel" aria-label="Tags for this image">
|
<aside class="fc-tag-panel" aria-label="Tags for this image">
|
||||||
<h3 class="fc-tag-panel__title">Tags</h3>
|
<h3 class="fc-tag-panel__title">Tags</h3>
|
||||||
<div class="fc-tag-panel__chips">
|
<div class="fc-tag-panel__chips">
|
||||||
<!-- #711: the kebab + its menu used to be NESTED inside the v-chip, which
|
<TagChip
|
||||||
swallowed/mis-routed the click and mis-anchored the (teleported) menu
|
|
||||||
so it never opened. Render the kebab as a SIBLING of the chip and use
|
|
||||||
the standard v-menu activator slot — Vuetify wires the click and
|
|
||||||
stacks the overlay above the modal natively. -->
|
|
||||||
<span
|
|
||||||
v-for="tag in modal.current?.tags || []"
|
v-for="tag in modal.current?.tags || []"
|
||||||
:key="tag.id" class="fc-tag-panel__chip"
|
:key="tag.id" :tag="tag"
|
||||||
>
|
@remove="onRemove" @rename="openRename" @set-fandom="openSetFandom"
|
||||||
<v-chip
|
/>
|
||||||
size="small" closable
|
|
||||||
:color="store.colorFor(tag.kind)" variant="tonal"
|
|
||||||
@click:close="onRemove(tag.id)"
|
|
||||||
>
|
|
||||||
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
|
||||||
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
|
||||||
</v-chip>
|
|
||||||
<v-menu location="bottom end">
|
|
||||||
<template #activator="{ props }">
|
|
||||||
<v-btn
|
|
||||||
v-bind="props" icon="mdi-dots-vertical" size="x-small"
|
|
||||||
variant="text" density="comfortable"
|
|
||||||
class="fc-tag-panel__kebab" @click.stop
|
|
||||||
/>
|
|
||||||
</template>
|
|
||||||
<v-list density="compact">
|
|
||||||
<v-list-item @click="openRename(tag)">
|
|
||||||
<v-list-item-title>Rename…</v-list-item-title>
|
|
||||||
</v-list-item>
|
|
||||||
<v-list-item
|
|
||||||
v-if="tag.kind === 'character'" @click="openSetFandom(tag)"
|
|
||||||
>
|
|
||||||
<v-list-item-title>Set fandom…</v-list-item-title>
|
|
||||||
</v-list-item>
|
|
||||||
</v-list>
|
|
||||||
</v-menu>
|
|
||||||
</span>
|
|
||||||
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -77,23 +45,15 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useModalStore } from '../../stores/modal.js'
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import { useTagStore } from '../../stores/tags.js'
|
import TagChip from './TagChip.vue'
|
||||||
import TagAutocomplete from './TagAutocomplete.vue'
|
import TagAutocomplete from './TagAutocomplete.vue'
|
||||||
import SuggestionsPanel from './SuggestionsPanel.vue'
|
import SuggestionsPanel from './SuggestionsPanel.vue'
|
||||||
import TagRenameDialog from './TagRenameDialog.vue'
|
import TagRenameDialog from './TagRenameDialog.vue'
|
||||||
import FandomSetDialog from './FandomSetDialog.vue'
|
import FandomSetDialog from './FandomSetDialog.vue'
|
||||||
|
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const store = useTagStore()
|
|
||||||
const errorMsg = ref(null)
|
const errorMsg = ref(null)
|
||||||
|
|
||||||
const KIND_ICONS = {
|
|
||||||
general: 'mdi-tag', character: 'mdi-account-circle',
|
|
||||||
fandom: 'mdi-book-open-page-variant', series: 'mdi-bookshelf',
|
|
||||||
meta: 'mdi-cog-outline', rating: 'mdi-shield-check-outline'
|
|
||||||
}
|
|
||||||
function iconFor(k) { return KIND_ICONS[k] || 'mdi-tag' }
|
|
||||||
|
|
||||||
async function onRemove(tagId) {
|
async function onRemove(tagId) {
|
||||||
errorMsg.value = null
|
errorMsg.value = null
|
||||||
try { await modal.removeTag(tagId) }
|
try { await modal.removeTag(tagId) }
|
||||||
@@ -148,7 +108,4 @@ async function onFandomUpdated() {
|
|||||||
margin-bottom: 12px;
|
margin-bottom: 12px;
|
||||||
}
|
}
|
||||||
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||||
.fc-tag-panel__chip { display: inline-flex; align-items: center; gap: 1px; }
|
|
||||||
.fc-tag-panel__kebab { opacity: 0.7; }
|
|
||||||
.fc-tag-panel__chip:hover .fc-tag-panel__kebab { opacity: 1; }
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -185,13 +185,9 @@
|
|||||||
:loading="normCommitting"
|
:loading="normCommitting"
|
||||||
@click="onNormCommit"
|
@click="onNormCommit"
|
||||||
>Standardize {{ normPreview.total_changes }} tag group(s)</v-btn>
|
>Standardize {{ normPreview.total_changes }} tag group(s)</v-btn>
|
||||||
<span
|
<span v-if="normResult === 'queued'" class="ml-3 text-caption text-success">
|
||||||
v-if="normResult"
|
Queued ✓ — runs in the background (you can leave this page). It
|
||||||
class="ml-3 text-caption"
|
processes in chunks, so re-run “Preview” later to confirm it's all done.
|
||||||
:class="normResult === 'ok' ? 'text-success' : 'text-warning'"
|
|
||||||
>
|
|
||||||
<template v-if="normResult === 'ok'">Standardization complete ✓</template>
|
|
||||||
<template v-else>Finished with status: {{ normResult }}</template>
|
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
@@ -289,13 +285,12 @@ async function onNormCommit() {
|
|||||||
normCommitting.value = true
|
normCommitting.value = true
|
||||||
normResult.value = null
|
normResult.value = null
|
||||||
try {
|
try {
|
||||||
// Long op (FK repoints): enqueue the maintenance task, then tail the
|
// Fire-and-forget: the task is time-boxed and self-resuming across chunks
|
||||||
// activity dashboard until its row reaches a terminal status. (The
|
// (a large back-catalog can't finish in one run), so we DON'T poll-until-
|
||||||
// per-run summary dict isn't exposed by /activity/runs, so we surface
|
// done — that would falsely report "complete" after the first chunk. Just
|
||||||
// the terminal status — the dry-run preview is the detailed view.)
|
// confirm it's queued; the operator can re-run Preview later to verify.
|
||||||
const { task_id: taskId } = await store.normalizeTags({ dryRun: false })
|
await store.normalizeTags({ dryRun: false })
|
||||||
const row = await store.pollTaskUntilDone(taskId)
|
normResult.value = 'queued'
|
||||||
normResult.value = row?.status || 'ok'
|
|
||||||
normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] }
|
normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] }
|
||||||
} finally {
|
} finally {
|
||||||
normCommitting.value = false
|
normCommitting.value = false
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
"""Queue routing — the long one-shot maintenance tasks run on a dedicated
|
||||||
|
`maintenance_long` lane so a 30-min backup or a multi-chunk audit can't starve
|
||||||
|
the quick recovery sweeps / vacuum on the concurrency-1 `maintenance` lane
|
||||||
|
(operator-flagged 2026-06-07)."""
|
||||||
|
from backend.app.celery_app import celery
|
||||||
|
|
||||||
|
|
||||||
|
def test_long_one_shots_route_to_maintenance_long():
|
||||||
|
routes = celery.conf.task_routes
|
||||||
|
for prefix in (
|
||||||
|
"backend.app.tasks.backup.*",
|
||||||
|
"backend.app.tasks.admin.*",
|
||||||
|
"backend.app.tasks.library_audit.*",
|
||||||
|
):
|
||||||
|
assert routes[prefix]["queue"] == "maintenance_long"
|
||||||
|
|
||||||
|
|
||||||
|
def test_quick_maintenance_stays_on_maintenance():
|
||||||
|
routes = celery.conf.task_routes
|
||||||
|
assert routes["backend.app.tasks.maintenance.*"]["queue"] == "maintenance"
|
||||||
@@ -123,6 +123,29 @@ async def test_live_merges_case_variants_and_repoints_images(db):
|
|||||||
assert assoc == 3
|
assert assoc == 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_time_budget_stops_partial_and_reports_remaining(db):
|
||||||
|
"""A zero budget stops before the first group so a huge back-catalog can't
|
||||||
|
run the task into the Celery time limit; it reports partial/remaining so the
|
||||||
|
caller re-enqueues to continue (operator-flagged 40-min timeout 2026-06-07)."""
|
||||||
|
await _raw_tag(db, "alpha", TagKind.general)
|
||||||
|
await _raw_tag(db, "beta", TagKind.general)
|
||||||
|
|
||||||
|
summary = await normalize_existing_tags(db, dry_run=False, time_budget_seconds=0)
|
||||||
|
|
||||||
|
assert summary["partial"] is True
|
||||||
|
assert summary["groups_processed"] == 0
|
||||||
|
assert summary["remaining"] == summary["total_changes"] >= 2
|
||||||
|
# Nothing recased yet — the budget cut before any work.
|
||||||
|
assert await _count_named(db, "alpha", TagKind.general) == 1
|
||||||
|
|
||||||
|
# A full (unbudgeted) run finishes and reports nothing remaining.
|
||||||
|
done = await normalize_existing_tags(db, dry_run=False)
|
||||||
|
assert done["partial"] is False
|
||||||
|
assert done["remaining"] == 0
|
||||||
|
assert await _count_named(db, "Alpha", TagKind.general) == 1
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_idempotent_second_run_is_noop(db):
|
async def test_idempotent_second_run_is_noop(db):
|
||||||
await _raw_tag(db, "kafka", TagKind.character)
|
await _raw_tag(db, "kafka", TagKind.character)
|
||||||
|
|||||||
@@ -69,6 +69,60 @@ def test_scan_library_for_rule_populates_matched_ids_for_transparency(
|
|||||||
assert status == "ready"
|
assert status == "ready"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_time_boxes_and_reenqueues(db_sync, tmp_path, monkeypatch):
|
||||||
|
"""A zero chunk budget stops before scanning and re-enqueues to continue,
|
||||||
|
leaving the audit 'running' — so a huge library can't run the task into the
|
||||||
|
Celery limit or hog the maintenance queue (operator-flagged 2026-06-07)."""
|
||||||
|
_mk_image(db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="a.png")
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.5},
|
||||||
|
status="running", matched_ids=[],
|
||||||
|
)
|
||||||
|
db_sync.add(audit)
|
||||||
|
db_sync.commit()
|
||||||
|
audit_id = audit.id
|
||||||
|
|
||||||
|
from backend.app.tasks import library_audit as la
|
||||||
|
delays = []
|
||||||
|
monkeypatch.setattr(la.scan_library_for_rule, "delay", lambda aid: delays.append(aid))
|
||||||
|
monkeypatch.setattr(la, "_CHUNK_SECONDS", 0) # time-box on the first iteration
|
||||||
|
la.scan_library_for_rule.run(audit_id)
|
||||||
|
|
||||||
|
status = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert status == "running" # not finished — handed off to the next chunk
|
||||||
|
assert delays == [audit_id] # re-enqueued itself
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_resumes_and_accumulates_matched(db_sync, tmp_path, monkeypatch):
|
||||||
|
"""A later chunk keeps the prior chunks' matches and appends its own."""
|
||||||
|
trans, _ = _mk_image(
|
||||||
|
db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="t.png",
|
||||||
|
)
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.5},
|
||||||
|
status="running", matched_ids=[424242], resume_after_id=0,
|
||||||
|
)
|
||||||
|
db_sync.add(audit)
|
||||||
|
db_sync.commit()
|
||||||
|
audit_id = audit.id
|
||||||
|
|
||||||
|
from backend.app.tasks import library_audit as la
|
||||||
|
monkeypatch.setattr(la.scan_library_for_rule, "delay", lambda aid: None)
|
||||||
|
la.scan_library_for_rule.run(audit_id)
|
||||||
|
|
||||||
|
matched = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
status = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert status == "ready"
|
||||||
|
assert 424242 in matched # carried over from a prior chunk
|
||||||
|
assert trans.id in matched # found this chunk
|
||||||
|
|
||||||
|
|
||||||
def test_scan_library_for_rule_skips_missing_files_gracefully(
|
def test_scan_library_for_rule_skips_missing_files_gracefully(
|
||||||
db_sync, tmp_path, monkeypatch,
|
db_sync, tmp_path, monkeypatch,
|
||||||
):
|
):
|
||||||
|
|||||||
Reference in New Issue
Block a user