feat(maintenance): scheduled + manual DB VACUUM ANALYZE + bloat readout
The TABLESAMPLE showcase reads physical blocks (bloat-sensitive), and the periodic prune/backfill/recovery tasks churn dead tuples faster than autovacuum always keeps up — so explicit maintenance earns its keep here. - tasks.maintenance.vacuum_analyze: VACUUM (ANALYZE) over high-churn tables (VACUUM_TABLES) on an AUTOCOMMIT connection (VACUUM can't run in a txn). Scheduled weekly via Beat; also operator-triggerable. - _sync_engine.get_sync_engine(): expose the process engine for the autocommit connection. - GET /api/admin/maintenance/db-stats: per-table n_live/n_dead/dead_pct + last (auto)vacuum/analyze from pg_stat_user_tables — visibility, not a black box. - POST /api/admin/maintenance/vacuum: enqueue the task on demand. Tests: vacuum task runs + reports tables; db-stats shape; trigger queues. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -34,3 +34,10 @@ def sync_session_factory():
|
||||
)
|
||||
_SESSIONMAKER = sessionmaker(_ENGINE, expire_on_commit=False)
|
||||
return _SESSIONMAKER
|
||||
|
||||
|
||||
def get_sync_engine():
|
||||
"""The process-wide sync Engine — for raw work that needs a connection
|
||||
directly (e.g. AUTOCOMMIT VACUUM, which can't run inside a transaction)."""
|
||||
sync_session_factory() # ensure _ENGINE is initialized
|
||||
return _ENGINE
|
||||
|
||||
@@ -22,10 +22,29 @@ from ..models import (
|
||||
TaskRun,
|
||||
)
|
||||
from ..utils.phash import compute_phash
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
from ._sync_engine import (
|
||||
get_sync_engine,
|
||||
sync_session_factory as _sync_session_factory,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# High-churn tables whose dead-tuple bloat matters: the TABLESAMPLE showcase
|
||||
# reads physical blocks (bloat slows it directly), and the periodic
|
||||
# prune/backfill/recovery tasks generate dead tuples faster than autovacuum
|
||||
# always keeps up with. VACUUM reclaims them; ANALYZE refreshes planner stats.
|
||||
# Allowlist ONLY — names are interpolated into VACUUM, so they must never come
|
||||
# from request input.
|
||||
VACUUM_TABLES = (
|
||||
"image_record",
|
||||
"image_provenance",
|
||||
"post_attachment",
|
||||
"download_event",
|
||||
"task_run",
|
||||
"import_task",
|
||||
"import_batch",
|
||||
)
|
||||
|
||||
STUCK_THRESHOLD_MINUTES = 5
|
||||
# Archive ImportTasks run the per-member pipeline inline for every
|
||||
# member (import_archive_file: soft=30min/hard=35min). The ImportTask
|
||||
@@ -761,3 +780,20 @@ def cleanup_old_download_events() -> int:
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.vacuum_analyze")
|
||||
def vacuum_analyze() -> dict:
|
||||
"""Periodic VACUUM (ANALYZE) over the high-churn tables (VACUUM_TABLES) to
|
||||
reclaim dead-tuple bloat and refresh planner statistics. VACUUM cannot run
|
||||
inside a transaction block, so it runs on an AUTOCOMMIT connection.
|
||||
Scheduled weekly; also operator-triggerable from Settings → Maintenance.
|
||||
"""
|
||||
engine = get_sync_engine()
|
||||
done = []
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
||||
for table in VACUUM_TABLES:
|
||||
conn.exec_driver_sql(f"VACUUM (ANALYZE) {table}")
|
||||
done.append(table)
|
||||
log.info("vacuum_analyze complete: %s", done)
|
||||
return {"vacuumed": done}
|
||||
|
||||
Reference in New Issue
Block a user