diff --git a/README.md b/README.md index e8204d4..e909d4c 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,11 @@ fingerprint, under `media/`. It never deletes from the bucket; the bucket's own retention thins old dumps. The first copy of a large library takes hours; a restart along the way carries on where it stopped rather than starting over. +Once a week the bucket is checked against what was sent, and anything missing +goes again in the next run. When the newest database backup is more than a day +old or failed, or the check found files missing, the **Backups** row on +**Settings → System** and the dot beside the brand turn amber and say why. + To rebuild after losing the server, start a fresh install (first boot as above), enter the same destination, and under **Restore from the bucket** choose a dump. The database is replaced, then every file is fetched back to its place diff --git a/backend/app/api/system_backup.py b/backend/app/api/system_backup.py index 42f05af..0fad222 100644 --- a/backend/app/api/system_backup.py +++ b/backend/app/api/system_backup.py @@ -20,7 +20,7 @@ system_backup_bp = Blueprint( "system_backup", __name__, url_prefix="/api/system/backup", ) -_KINDS = frozenset({"db", "images", "media", "restore"}) +_KINDS = frozenset({"db", "images", "media", "restore", "reconcile"}) _TAG_MAX_LEN = 64 _BACKUP_SETTINGS_FIELDS = ( "backup_db_nightly_enabled", diff --git a/backend/app/api/system_health.py b/backend/app/api/system_health.py index 6fffe84..1206ef6 100644 --- a/backend/app/api/system_health.py +++ b/backend/app/api/system_health.py @@ -35,6 +35,7 @@ from sqlalchemy import select, text from ..config import get_config from ..extensions import get_session from ..models import ServiceSeen +from ..services import backup_health from ..services.worker_lanes import SWEEP_PERIOD_SECONDS system_health_bp = Blueprint("system_health", __name__, url_prefix="/api/system") @@ -227,6 +228,17 @@ async def system_health(): **{k: v for k, v in (row.details or {}).items() if k != "agent_id"}, }) + # Whether the backups are keeping up. Read like the rest, and a + # failure to read it is reported as unknown rather than raised. + try: + backup = await session.run_sync(lambda s: backup_health.backup_part(s, now)) + except Exception as exc: # noqa: BLE001 — a probe reports, it never raises + backup = {"key": "backup", "kind": "backup", "name": "Backups", + "state": _UNKNOWN, + "detail": f"could not be checked: {type(exc).__name__}"} + if backup is not None: + parts.append(backup) + parts.append(await _probe_redis()) overall = max((p["state"] for p in parts), key=lambda s: _SEVERITY[s], default=_UNKNOWN) diff --git a/backend/app/services/backup_health.py b/backend/app/services/backup_health.py new file mode 100644 index 0000000..130c89b --- /dev/null +++ b/backend/app/services/backup_health.py @@ -0,0 +1,120 @@ +"""Are the backups keeping up? One row on the System view, and the nav dot. + +The backups run on their own at night, so the only way to notice one has +stopped is to be told. The System view's health endpoint asks `backup_part`, +which answers with a part like any other: `ok`, or `stale` with the sentence +that says what is wrong. It warns when + +- the newest database backup is more than a day old, or there is none; +- the newest database backup failed; +- the weekly check of the off-site bucket found files missing from it, and no + media run has finished since to send them again. + +No part at all when backups aren't set up (no off-site destination and the +nightly backup off): someone who chose not to back up is not told so every +fifteen seconds. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from sqlalchemy import select + +from ..models import BackupRun, ImportSettings +from . import backup_destination + +# A nightly backup is due every 24 hours. The hour of slack covers the dump +# itself and a beat tick that lands a little late. +DUMP_DUE_SECONDS = 25 * 3600 + + +@dataclass(frozen=True) +class BackupFacts: + expected: bool + offsite: bool + last_dump_at: datetime | None = None + last_dump_error: str | None = None + missing: int = 0 + + +def _age(seconds: float) -> str: + hours = int(seconds // 3600) + if hours >= 48: + return f"{hours // 24} days" + if hours >= 1: + return f"{hours} h" + return f"{max(1, int(seconds // 60))} min" + + +def verdict(facts: BackupFacts, now: datetime) -> dict | None: + """The part for these facts, or None when backups aren't set up.""" + if not facts.expected: + return None + where = "to the off-site bucket" if facts.offsite else "on this server" + problems = [] + if facts.last_dump_error: + problems.append(f"The last database backup failed: {facts.last_dump_error}") + if facts.last_dump_at is None: + if not facts.last_dump_error: + problems.append(f"No database backup has been taken {where} yet") + else: + age = (now - facts.last_dump_at).total_seconds() + if age >= DUMP_DUE_SECONDS: + problems.append(f"The newest database backup {where} is {_age(age)} old") + if facts.missing: + problems.append( + f"The bucket check found {facts.missing:,} file(s) missing from the bucket; " + "the next media run sends them again" + ) + part = {"key": "backup", "kind": "backup", "name": "Backups"} + if problems: + return {**part, "state": "stale", "detail": ". ".join(problems)} + age = (now - facts.last_dump_at).total_seconds() + return {**part, "state": "ok", + "detail": f"The newest database backup {where} is {_age(age)} old"} + + +def gather(session) -> BackupFacts: + """Read the facts `verdict` judges, from a sync session.""" + settings = ImportSettings.load_sync(session) + offsite = backup_destination.describe(settings)["configured"] + expected = offsite or bool(settings.backup_db_nightly_enabled) + if not expected: + return BackupFacts(expected=False, offsite=False) + + def newest(kind: str, statuses: tuple[str, ...]): + return session.execute( + select(BackupRun).where(BackupRun.kind == kind) + .where(BackupRun.status.in_(statuses)) + .order_by(BackupRun.started_at.desc()).limit(1) + ).scalar_one_or_none() + + last_ok = newest("db", ("ok",)) + last_done = newest("db", ("ok", "error")) + failed = None + if last_done is not None and last_done.status == "error": + failed = last_done.error or "no reason recorded" + + missing = 0 + if offsite: + check = newest("reconcile", ("ok",)) + dropped = int((check.manifest or {}).get("dropped") or 0) if check else 0 + if dropped: + sent_since = session.execute( + select(BackupRun.id).where(BackupRun.kind == "media") + .where(BackupRun.status == "ok") + .where(BackupRun.finished_at > check.finished_at).limit(1) + ).scalar_one_or_none() + missing = 0 if sent_since is not None else dropped + + return BackupFacts( + expected=True, offsite=offsite, + last_dump_at=(last_ok.finished_at or last_ok.started_at) if last_ok else None, + last_dump_error=failed, missing=missing, + ) + + +def backup_part(session, now: datetime) -> dict | None: + return verdict(gather(session), now) diff --git a/backend/app/tasks/backup.py b/backend/app/tasks/backup.py index 33d0866..0e42aea 100644 --- a/backend/app/tasks/backup.py +++ b/backend/app/tasks/backup.py @@ -280,9 +280,10 @@ def prune_backups() -> dict: """Daily Beat. Per-kind retention from ImportSettings. Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L, - "restore_deleted": R, "files_unlinked": K}. Media and restore runs leave - no files here; only their history is trimmed, to the latest - MEDIA_RUNS_KEPT and RESTORE_RUNS_KEPT. + "restore_deleted": R, "reconcile_deleted": C, "files_unlinked": K}. Media, + restore and reconcile runs leave no files here; only their history is + trimmed, to the latest MEDIA_RUNS_KEPT, RESTORE_RUNS_KEPT and + RECONCILE_RUNS_KEPT. Tagged rows (tag IS NOT NULL) are never pruned. Status='running' / 'restoring' rows are never pruned — the recover_stalled_backup_runs sweep flips truly-stuck ones to @@ -292,7 +293,7 @@ def prune_backups() -> dict: """ SessionLocal = _sync_session_factory() counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0, - "restore_deleted": 0, "files_unlinked": 0} + "restore_deleted": 0, "reconcile_deleted": 0, "files_unlinked": 0} with SessionLocal() as session: s = ImportSettings.load_sync(session) for kind, keep in ( @@ -300,6 +301,7 @@ def prune_backups() -> dict: ("images", s.backup_images_keep_last_n), ("media", MEDIA_RUNS_KEPT), ("restore", RESTORE_RUNS_KEPT), + ("reconcile", RECONCILE_RUNS_KEPT), ): candidates = session.execute( select(BackupRun) @@ -365,6 +367,8 @@ MEDIA_CHUNK_SECONDS = 45 * 60 MEDIA_RUNS_KEPT = 60 # A failed file is listed on the run, up to this many; the count covers all. MEDIA_FAILURES_LISTED = 20 +# The weekly bucket check keeps about a quarter's history. +RECONCILE_RUNS_KEPT = 12 def _offsite_manifest(target) -> dict: @@ -585,7 +589,11 @@ def reconcile_media_backup() -> dict: """Weekly Beat. Compare what the off-site bucket holds under `media/` with what FabledCurator believes it sent: a missing object is sent again by the next run, and an object already there is not sent twice. Skipped - while a media run is going, and when no destination is set up.""" + while a media run is going, and when no destination is set up. + + Each check is kept as a run of kind "reconcile" with its counts + ({listed, dropped, adopted}), so the Backups card lists it and the System + view can warn when files were missing from the bucket.""" SessionLocal = _sync_session_factory() with SessionLocal() as session: target = backup_destination.saved_target(ImportSettings.load_sync(session)) @@ -597,7 +605,27 @@ def reconcile_media_backup() -> dict: ).scalar_one_or_none() if going is not None: return {"skipped": "a media backup is running"} - return backup_media.reconcile(session, target) + now = datetime.now(UTC) + row = BackupRun(kind="reconcile", status="running", triggered_by="weekly", + started_at=now, last_progress_at=now, + manifest={"offsite": _offsite_manifest(target)}) + session.add(row) + session.commit() + run_id = row.id + try: + counts = backup_media.reconcile(session, target) + except (SoftTimeLimitExceeded, Exception) as exc: + session.rollback() + row = session.get(BackupRun, run_id) + if row is not None: + _mark_failed(session, row, exc) + raise + row = session.get(BackupRun, run_id) + row.status = "ok" + row.finished_at = datetime.now(UTC) + row.manifest = {**(row.manifest or {}), **counts} + session.commit() + return {"backup_run_id": run_id, **counts} # ── Restoring from the off-site bucket ─────────────────────────────────────── diff --git a/frontend/src/components/settings/BackupCard.vue b/frontend/src/components/settings/BackupCard.vue index 13660d1..c88eea5 100644 --- a/frontend/src/components/settings/BackupCard.vue +++ b/frontend/src/components/settings/BackupCard.vue @@ -25,6 +25,21 @@ {{ store.lastError }} + + {{ backupPart.detail }}. +
+ mdi-check-circle + {{ backupPart.detail }}. +
+

Database

@@ -93,8 +108,12 @@ @click="onTrigger('media')" >Send new files now - A run is going: {{ runningSent.toLocaleString() }} files sent so far. + A run is going: {{ runningSent.toLocaleString() }} files + ({{ humanBytes(runningMedia.size_bytes || 0) }}) sent so far. +

+ {{ lastCheckLine }} +

@@ -223,13 +242,32 @@ import { computed, onMounted, onUnmounted, ref, watch } from 'vue' import { useBackupStore } from '../../stores/backup.js' +import { useSystemHealthStore } from '../../stores/systemHealth.js' import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue' import MaintenanceTile from '../common/MaintenanceTile.vue' import { humanBytes } from '../../utils/bytes.js' -import { formatDateTime } from '../../utils/date.js' +import { formatDateTime, formatRelative } from '../../utils/date.js' import BackupRunsTable from './BackupRunsTable.vue' const store = useBackupStore() +const healthStore = useSystemHealthStore() + +// The server's verdict on the backups; absent when none are set up. +const backupPart = computed(() => healthStore.parts.find((p) => p.key === 'backup')) +const newestDump = computed(() => store.dbRuns.find((r) => r.status === 'ok')) + +// The weekly check of the bucket against what was sent. +const lastCheckLine = computed(() => { + const r = store.reconcileRuns.find((x) => x.status !== 'running') + if (!r) return 'The bucket is checked against what was sent once a week; no check has run yet.' + const when = formatRelative(r.finished_at || r.started_at) + if (r.status === 'error') return `The last bucket check (${when}) failed: ${r.error}` + const n = (k) => (r.manifest?.[k] || 0).toLocaleString() + const missing = r.manifest?.dropped + ? `${n('dropped')} missing, sent again by the next run` + : 'none missing' + return `Bucket checked ${when}: ${n('listed')} files there, ${missing}.` +}) const settings = computed(() => store.settings || { backup_db_nightly_enabled: false, @@ -345,7 +383,10 @@ function pollOnce() { if (offsite.value) { store.loadRuns('media') store.loadRuns('restore') - if (polls++ % COVERAGE_EVERY === 0) store.loadMediaCoverage() + if (polls++ % COVERAGE_EVERY === 0) { + store.loadMediaCoverage() + store.loadRuns('reconcile') + } } else { store.loadRuns('images') } @@ -353,6 +394,7 @@ function pollOnce() { onMounted(() => { store.loadSettings() + healthStore.refresh() if (!store.destination) store.loadDestination().catch(() => {}) pollOnce() pollId = setInterval(pollOnce, 5000) @@ -367,6 +409,7 @@ async function onTrigger(kind) { try { await store.triggerBackup(kind) pollOnce() + healthStore.refresh() } finally { triggeringRef.value = false } diff --git a/frontend/src/components/settings/BackupRunsTable.vue b/frontend/src/components/settings/BackupRunsTable.vue index ba2315f..aac5ce5 100644 --- a/frontend/src/components/settings/BackupRunsTable.vue +++ b/frontend/src/components/settings/BackupRunsTable.vue @@ -12,7 +12,8 @@ - +