fix(tags): time-box + self-resume the tag standardization (stop the 40-min timeout)
normalize_tags_task timed out at the 40-min hard limit on a large back-catalog
(the first run recases the whole booru vocabulary) — operator-flagged, and it
monopolized the concurrency-1 maintenance queue while doing so.
normalize_existing_tags now takes time_budget_seconds: the live run stops
cleanly at the budget and reports {partial, remaining}. The task runs 600s
chunks and re-enqueues itself until nothing remains (idempotent — commits per
group, so the next chunk skips already-canonical groups). Short chunks let the
recovery sweep and other maintenance tasks interleave instead of being blocked
for 40 minutes.
Frontend: the Standardize button is now fire-and-forget ("Queued — runs in the
background; re-run Preview to confirm") instead of poll-until-done, which would
have falsely reported "complete" after the first chunk.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -82,11 +82,21 @@ def reextract_archive_attachments_task(self) -> dict:
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
# Time-box one chunk well under the soft limit so a large back-catalog (the
|
||||
# first run recases the whole booru vocabulary) can't run the task into the
|
||||
# Celery time limit — it timed out at 40 min, operator-flagged 2026-06-07. The
|
||||
# task re-enqueues itself until nothing remains (idempotent — already-canonical
|
||||
# groups are skipped). 600s keeps each chunk short enough that the recovery
|
||||
# sweep and other maintenance tasks interleave on the concurrency-1 queue.
|
||||
_NORMALIZE_CHUNK_SECONDS = 600
|
||||
|
||||
|
||||
def normalize_tags_task(self) -> dict:
|
||||
"""Wraps tag_service.normalize_existing_tags (#714): Title-Case the
|
||||
back-catalog and merge case/whitespace-variant duplicate tags via the
|
||||
tested async merge path. Runs under its own asyncio loop + per-task async
|
||||
engine (NullPool, disposed when the loop ends), mirroring download_source."""
|
||||
tested async merge path. Time-boxed + self-resuming so a huge first run
|
||||
finishes across chunks instead of timing out. Runs under its own asyncio
|
||||
loop + per-task async engine (NullPool), mirroring download_source."""
|
||||
import asyncio
|
||||
|
||||
from ..services.tag_service import normalize_existing_tags
|
||||
@@ -97,8 +107,20 @@ def normalize_tags_task(self) -> dict:
|
||||
try:
|
||||
async with async_factory() as session:
|
||||
# normalize_existing_tags commits per group internally.
|
||||
return await normalize_existing_tags(session, dry_run=False)
|
||||
return await normalize_existing_tags(
|
||||
session, dry_run=False,
|
||||
time_budget_seconds=_NORMALIZE_CHUNK_SECONDS,
|
||||
)
|
||||
finally:
|
||||
await async_engine.dispose()
|
||||
|
||||
return asyncio.run(_run())
|
||||
summary = asyncio.run(_run())
|
||||
# More groups to canonicalize than fit this chunk — continue in the next.
|
||||
if summary.get("partial") and summary.get("remaining", 0) > 0:
|
||||
log.info(
|
||||
"normalize_tags_task chunk done (%d processed, %d remaining) — "
|
||||
"re-enqueuing to continue",
|
||||
summary.get("groups_processed", 0), summary["remaining"],
|
||||
)
|
||||
normalize_tags_task.delay()
|
||||
return summary
|
||||
|
||||
Reference in New Issue
Block a user