feat: the Backups screen says whether backups are keeping up, and the System view warns when they aren't
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m54s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m54s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
- A Backups row on Settings → System and the nav dot turn amber when the newest database dump is more than a day old or failed, or the weekly bucket check found files missing that no media run has sent since. No row when backups aren't set up. - The weekly bucket check is kept as a run (kind "reconcile") with its counts, trimmed to the last 12, and listed on the Backups card. - The Backups card leads with the same verdict and the newest dump's size, shows bytes beside the running media job's file count, and lets each run open its list of failed files with why. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -192,6 +192,11 @@ fingerprint, under `media/`. It never deletes from the bucket; the bucket's own
|
|||||||
retention thins old dumps. The first copy of a large library takes hours; a
|
retention thins old dumps. The first copy of a large library takes hours; a
|
||||||
restart along the way carries on where it stopped rather than starting over.
|
restart along the way carries on where it stopped rather than starting over.
|
||||||
|
|
||||||
|
Once a week the bucket is checked against what was sent, and anything missing
|
||||||
|
goes again in the next run. When the newest database backup is more than a day
|
||||||
|
old or failed, or the check found files missing, the **Backups** row on
|
||||||
|
**Settings → System** and the dot beside the brand turn amber and say why.
|
||||||
|
|
||||||
To rebuild after losing the server, start a fresh install (first boot as
|
To rebuild after losing the server, start a fresh install (first boot as
|
||||||
above), enter the same destination, and under **Restore from the bucket** choose
|
above), enter the same destination, and under **Restore from the bucket** choose
|
||||||
a dump. The database is replaced, then every file is fetched back to its place
|
a dump. The database is replaced, then every file is fetched back to its place
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ system_backup_bp = Blueprint(
|
|||||||
"system_backup", __name__, url_prefix="/api/system/backup",
|
"system_backup", __name__, url_prefix="/api/system/backup",
|
||||||
)
|
)
|
||||||
|
|
||||||
_KINDS = frozenset({"db", "images", "media", "restore"})
|
_KINDS = frozenset({"db", "images", "media", "restore", "reconcile"})
|
||||||
_TAG_MAX_LEN = 64
|
_TAG_MAX_LEN = 64
|
||||||
_BACKUP_SETTINGS_FIELDS = (
|
_BACKUP_SETTINGS_FIELDS = (
|
||||||
"backup_db_nightly_enabled",
|
"backup_db_nightly_enabled",
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ from sqlalchemy import select, text
|
|||||||
from ..config import get_config
|
from ..config import get_config
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import ServiceSeen
|
from ..models import ServiceSeen
|
||||||
|
from ..services import backup_health
|
||||||
from ..services.worker_lanes import SWEEP_PERIOD_SECONDS
|
from ..services.worker_lanes import SWEEP_PERIOD_SECONDS
|
||||||
|
|
||||||
system_health_bp = Blueprint("system_health", __name__, url_prefix="/api/system")
|
system_health_bp = Blueprint("system_health", __name__, url_prefix="/api/system")
|
||||||
@@ -227,6 +228,17 @@ async def system_health():
|
|||||||
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# Whether the backups are keeping up. Read like the rest, and a
|
||||||
|
# failure to read it is reported as unknown rather than raised.
|
||||||
|
try:
|
||||||
|
backup = await session.run_sync(lambda s: backup_health.backup_part(s, now))
|
||||||
|
except Exception as exc: # noqa: BLE001 — a probe reports, it never raises
|
||||||
|
backup = {"key": "backup", "kind": "backup", "name": "Backups",
|
||||||
|
"state": _UNKNOWN,
|
||||||
|
"detail": f"could not be checked: {type(exc).__name__}"}
|
||||||
|
if backup is not None:
|
||||||
|
parts.append(backup)
|
||||||
|
|
||||||
parts.append(await _probe_redis())
|
parts.append(await _probe_redis())
|
||||||
|
|
||||||
overall = max((p["state"] for p in parts), key=lambda s: _SEVERITY[s], default=_UNKNOWN)
|
overall = max((p["state"] for p in parts), key=lambda s: _SEVERITY[s], default=_UNKNOWN)
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""Are the backups keeping up? One row on the System view, and the nav dot.
|
||||||
|
|
||||||
|
The backups run on their own at night, so the only way to notice one has
|
||||||
|
stopped is to be told. The System view's health endpoint asks `backup_part`,
|
||||||
|
which answers with a part like any other: `ok`, or `stale` with the sentence
|
||||||
|
that says what is wrong. It warns when
|
||||||
|
|
||||||
|
- the newest database backup is more than a day old, or there is none;
|
||||||
|
- the newest database backup failed;
|
||||||
|
- the weekly check of the off-site bucket found files missing from it, and no
|
||||||
|
media run has finished since to send them again.
|
||||||
|
|
||||||
|
No part at all when backups aren't set up (no off-site destination and the
|
||||||
|
nightly backup off): someone who chose not to back up is not told so every
|
||||||
|
fifteen seconds.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..models import BackupRun, ImportSettings
|
||||||
|
from . import backup_destination
|
||||||
|
|
||||||
|
# A nightly backup is due every 24 hours. The hour of slack covers the dump
|
||||||
|
# itself and a beat tick that lands a little late.
|
||||||
|
DUMP_DUE_SECONDS = 25 * 3600
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class BackupFacts:
|
||||||
|
expected: bool
|
||||||
|
offsite: bool
|
||||||
|
last_dump_at: datetime | None = None
|
||||||
|
last_dump_error: str | None = None
|
||||||
|
missing: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def _age(seconds: float) -> str:
|
||||||
|
hours = int(seconds // 3600)
|
||||||
|
if hours >= 48:
|
||||||
|
return f"{hours // 24} days"
|
||||||
|
if hours >= 1:
|
||||||
|
return f"{hours} h"
|
||||||
|
return f"{max(1, int(seconds // 60))} min"
|
||||||
|
|
||||||
|
|
||||||
|
def verdict(facts: BackupFacts, now: datetime) -> dict | None:
|
||||||
|
"""The part for these facts, or None when backups aren't set up."""
|
||||||
|
if not facts.expected:
|
||||||
|
return None
|
||||||
|
where = "to the off-site bucket" if facts.offsite else "on this server"
|
||||||
|
problems = []
|
||||||
|
if facts.last_dump_error:
|
||||||
|
problems.append(f"The last database backup failed: {facts.last_dump_error}")
|
||||||
|
if facts.last_dump_at is None:
|
||||||
|
if not facts.last_dump_error:
|
||||||
|
problems.append(f"No database backup has been taken {where} yet")
|
||||||
|
else:
|
||||||
|
age = (now - facts.last_dump_at).total_seconds()
|
||||||
|
if age >= DUMP_DUE_SECONDS:
|
||||||
|
problems.append(f"The newest database backup {where} is {_age(age)} old")
|
||||||
|
if facts.missing:
|
||||||
|
problems.append(
|
||||||
|
f"The bucket check found {facts.missing:,} file(s) missing from the bucket; "
|
||||||
|
"the next media run sends them again"
|
||||||
|
)
|
||||||
|
part = {"key": "backup", "kind": "backup", "name": "Backups"}
|
||||||
|
if problems:
|
||||||
|
return {**part, "state": "stale", "detail": ". ".join(problems)}
|
||||||
|
age = (now - facts.last_dump_at).total_seconds()
|
||||||
|
return {**part, "state": "ok",
|
||||||
|
"detail": f"The newest database backup {where} is {_age(age)} old"}
|
||||||
|
|
||||||
|
|
||||||
|
def gather(session) -> BackupFacts:
|
||||||
|
"""Read the facts `verdict` judges, from a sync session."""
|
||||||
|
settings = ImportSettings.load_sync(session)
|
||||||
|
offsite = backup_destination.describe(settings)["configured"]
|
||||||
|
expected = offsite or bool(settings.backup_db_nightly_enabled)
|
||||||
|
if not expected:
|
||||||
|
return BackupFacts(expected=False, offsite=False)
|
||||||
|
|
||||||
|
def newest(kind: str, statuses: tuple[str, ...]):
|
||||||
|
return session.execute(
|
||||||
|
select(BackupRun).where(BackupRun.kind == kind)
|
||||||
|
.where(BackupRun.status.in_(statuses))
|
||||||
|
.order_by(BackupRun.started_at.desc()).limit(1)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
last_ok = newest("db", ("ok",))
|
||||||
|
last_done = newest("db", ("ok", "error"))
|
||||||
|
failed = None
|
||||||
|
if last_done is not None and last_done.status == "error":
|
||||||
|
failed = last_done.error or "no reason recorded"
|
||||||
|
|
||||||
|
missing = 0
|
||||||
|
if offsite:
|
||||||
|
check = newest("reconcile", ("ok",))
|
||||||
|
dropped = int((check.manifest or {}).get("dropped") or 0) if check else 0
|
||||||
|
if dropped:
|
||||||
|
sent_since = session.execute(
|
||||||
|
select(BackupRun.id).where(BackupRun.kind == "media")
|
||||||
|
.where(BackupRun.status == "ok")
|
||||||
|
.where(BackupRun.finished_at > check.finished_at).limit(1)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
missing = 0 if sent_since is not None else dropped
|
||||||
|
|
||||||
|
return BackupFacts(
|
||||||
|
expected=True, offsite=offsite,
|
||||||
|
last_dump_at=(last_ok.finished_at or last_ok.started_at) if last_ok else None,
|
||||||
|
last_dump_error=failed, missing=missing,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def backup_part(session, now: datetime) -> dict | None:
|
||||||
|
return verdict(gather(session), now)
|
||||||
@@ -280,9 +280,10 @@ def prune_backups() -> dict:
|
|||||||
"""Daily Beat. Per-kind retention from ImportSettings.
|
"""Daily Beat. Per-kind retention from ImportSettings.
|
||||||
|
|
||||||
Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
|
Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
|
||||||
"restore_deleted": R, "files_unlinked": K}. Media and restore runs leave
|
"restore_deleted": R, "reconcile_deleted": C, "files_unlinked": K}. Media,
|
||||||
no files here; only their history is trimmed, to the latest
|
restore and reconcile runs leave no files here; only their history is
|
||||||
MEDIA_RUNS_KEPT and RESTORE_RUNS_KEPT.
|
trimmed, to the latest MEDIA_RUNS_KEPT, RESTORE_RUNS_KEPT and
|
||||||
|
RECONCILE_RUNS_KEPT.
|
||||||
Tagged rows (tag IS NOT NULL) are never pruned.
|
Tagged rows (tag IS NOT NULL) are never pruned.
|
||||||
Status='running' / 'restoring' rows are never pruned — the
|
Status='running' / 'restoring' rows are never pruned — the
|
||||||
recover_stalled_backup_runs sweep flips truly-stuck ones to
|
recover_stalled_backup_runs sweep flips truly-stuck ones to
|
||||||
@@ -292,7 +293,7 @@ def prune_backups() -> dict:
|
|||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0,
|
counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0,
|
||||||
"restore_deleted": 0, "files_unlinked": 0}
|
"restore_deleted": 0, "reconcile_deleted": 0, "files_unlinked": 0}
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
s = ImportSettings.load_sync(session)
|
s = ImportSettings.load_sync(session)
|
||||||
for kind, keep in (
|
for kind, keep in (
|
||||||
@@ -300,6 +301,7 @@ def prune_backups() -> dict:
|
|||||||
("images", s.backup_images_keep_last_n),
|
("images", s.backup_images_keep_last_n),
|
||||||
("media", MEDIA_RUNS_KEPT),
|
("media", MEDIA_RUNS_KEPT),
|
||||||
("restore", RESTORE_RUNS_KEPT),
|
("restore", RESTORE_RUNS_KEPT),
|
||||||
|
("reconcile", RECONCILE_RUNS_KEPT),
|
||||||
):
|
):
|
||||||
candidates = session.execute(
|
candidates = session.execute(
|
||||||
select(BackupRun)
|
select(BackupRun)
|
||||||
@@ -365,6 +367,8 @@ MEDIA_CHUNK_SECONDS = 45 * 60
|
|||||||
MEDIA_RUNS_KEPT = 60
|
MEDIA_RUNS_KEPT = 60
|
||||||
# A failed file is listed on the run, up to this many; the count covers all.
|
# A failed file is listed on the run, up to this many; the count covers all.
|
||||||
MEDIA_FAILURES_LISTED = 20
|
MEDIA_FAILURES_LISTED = 20
|
||||||
|
# The weekly bucket check keeps about a quarter's history.
|
||||||
|
RECONCILE_RUNS_KEPT = 12
|
||||||
|
|
||||||
|
|
||||||
def _offsite_manifest(target) -> dict:
|
def _offsite_manifest(target) -> dict:
|
||||||
@@ -585,7 +589,11 @@ def reconcile_media_backup() -> dict:
|
|||||||
"""Weekly Beat. Compare what the off-site bucket holds under `media/` with
|
"""Weekly Beat. Compare what the off-site bucket holds under `media/` with
|
||||||
what FabledCurator believes it sent: a missing object is sent again by
|
what FabledCurator believes it sent: a missing object is sent again by
|
||||||
the next run, and an object already there is not sent twice. Skipped
|
the next run, and an object already there is not sent twice. Skipped
|
||||||
while a media run is going, and when no destination is set up."""
|
while a media run is going, and when no destination is set up.
|
||||||
|
|
||||||
|
Each check is kept as a run of kind "reconcile" with its counts
|
||||||
|
({listed, dropped, adopted}), so the Backups card lists it and the System
|
||||||
|
view can warn when files were missing from the bucket."""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
target = backup_destination.saved_target(ImportSettings.load_sync(session))
|
target = backup_destination.saved_target(ImportSettings.load_sync(session))
|
||||||
@@ -597,7 +605,27 @@ def reconcile_media_backup() -> dict:
|
|||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if going is not None:
|
if going is not None:
|
||||||
return {"skipped": "a media backup is running"}
|
return {"skipped": "a media backup is running"}
|
||||||
return backup_media.reconcile(session, target)
|
now = datetime.now(UTC)
|
||||||
|
row = BackupRun(kind="reconcile", status="running", triggered_by="weekly",
|
||||||
|
started_at=now, last_progress_at=now,
|
||||||
|
manifest={"offsite": _offsite_manifest(target)})
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
run_id = row.id
|
||||||
|
try:
|
||||||
|
counts = backup_media.reconcile(session, target)
|
||||||
|
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||||
|
session.rollback()
|
||||||
|
row = session.get(BackupRun, run_id)
|
||||||
|
if row is not None:
|
||||||
|
_mark_failed(session, row, exc)
|
||||||
|
raise
|
||||||
|
row = session.get(BackupRun, run_id)
|
||||||
|
row.status = "ok"
|
||||||
|
row.finished_at = datetime.now(UTC)
|
||||||
|
row.manifest = {**(row.manifest or {}), **counts}
|
||||||
|
session.commit()
|
||||||
|
return {"backup_run_id": run_id, **counts}
|
||||||
|
|
||||||
|
|
||||||
# ── Restoring from the off-site bucket ───────────────────────────────────────
|
# ── Restoring from the off-site bucket ───────────────────────────────────────
|
||||||
|
|||||||
@@ -25,6 +25,21 @@
|
|||||||
{{ store.lastError }}
|
{{ store.lastError }}
|
||||||
</v-alert>
|
</v-alert>
|
||||||
|
|
||||||
|
<!-- Whether the backups are keeping up: the same verdict as the Backups
|
||||||
|
row on the System tab and the dot beside the brand. -->
|
||||||
|
<v-alert
|
||||||
|
v-if="backupPart && backupPart.state !== 'ok'"
|
||||||
|
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||||
|
>{{ backupPart.detail }}.</v-alert>
|
||||||
|
<div
|
||||||
|
v-else-if="backupPart" class="d-flex align-center text-body-2 mb-3"
|
||||||
|
style="font-variant-numeric: tabular-nums;"
|
||||||
|
>
|
||||||
|
<v-icon size="small" color="success" class="mr-2">mdi-check-circle</v-icon>
|
||||||
|
{{ backupPart.detail }}<template v-if="newestDump?.size_bytes">,
|
||||||
|
{{ humanBytes(newestDump.size_bytes) }}</template>.
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Database section -->
|
<!-- Database section -->
|
||||||
<h3 class="fc-section-title">Database</h3>
|
<h3 class="fc-section-title">Database</h3>
|
||||||
<div class="fc-settings-row mb-3">
|
<div class="fc-settings-row mb-3">
|
||||||
@@ -93,8 +108,12 @@
|
|||||||
@click="onTrigger('media')"
|
@click="onTrigger('media')"
|
||||||
>Send new files now</v-btn>
|
>Send new files now</v-btn>
|
||||||
<span v-if="mediaRunning" class="text-caption fc-muted ml-2">
|
<span v-if="mediaRunning" class="text-caption fc-muted ml-2">
|
||||||
A run is going: {{ runningSent.toLocaleString() }} files sent so far.
|
A run is going: {{ runningSent.toLocaleString() }} files
|
||||||
|
({{ humanBytes(runningMedia.size_bytes || 0) }}) sent so far.
|
||||||
</span>
|
</span>
|
||||||
|
<p class="fc-muted text-caption mb-2" style="font-variant-numeric: tabular-nums;">
|
||||||
|
{{ lastCheckLine }}
|
||||||
|
</p>
|
||||||
<BackupRunsTable :runs="store.mediaRuns" />
|
<BackupRunsTable :runs="store.mediaRuns" />
|
||||||
|
|
||||||
<v-divider class="my-4" />
|
<v-divider class="my-4" />
|
||||||
@@ -223,13 +242,32 @@
|
|||||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||||
|
|
||||||
import { useBackupStore } from '../../stores/backup.js'
|
import { useBackupStore } from '../../stores/backup.js'
|
||||||
|
import { useSystemHealthStore } from '../../stores/systemHealth.js'
|
||||||
import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
import { humanBytes } from '../../utils/bytes.js'
|
import { humanBytes } from '../../utils/bytes.js'
|
||||||
import { formatDateTime } from '../../utils/date.js'
|
import { formatDateTime, formatRelative } from '../../utils/date.js'
|
||||||
import BackupRunsTable from './BackupRunsTable.vue'
|
import BackupRunsTable from './BackupRunsTable.vue'
|
||||||
|
|
||||||
const store = useBackupStore()
|
const store = useBackupStore()
|
||||||
|
const healthStore = useSystemHealthStore()
|
||||||
|
|
||||||
|
// The server's verdict on the backups; absent when none are set up.
|
||||||
|
const backupPart = computed(() => healthStore.parts.find((p) => p.key === 'backup'))
|
||||||
|
const newestDump = computed(() => store.dbRuns.find((r) => r.status === 'ok'))
|
||||||
|
|
||||||
|
// The weekly check of the bucket against what was sent.
|
||||||
|
const lastCheckLine = computed(() => {
|
||||||
|
const r = store.reconcileRuns.find((x) => x.status !== 'running')
|
||||||
|
if (!r) return 'The bucket is checked against what was sent once a week; no check has run yet.'
|
||||||
|
const when = formatRelative(r.finished_at || r.started_at)
|
||||||
|
if (r.status === 'error') return `The last bucket check (${when}) failed: ${r.error}`
|
||||||
|
const n = (k) => (r.manifest?.[k] || 0).toLocaleString()
|
||||||
|
const missing = r.manifest?.dropped
|
||||||
|
? `${n('dropped')} missing, sent again by the next run`
|
||||||
|
: 'none missing'
|
||||||
|
return `Bucket checked ${when}: ${n('listed')} files there, ${missing}.`
|
||||||
|
})
|
||||||
|
|
||||||
const settings = computed(() => store.settings || {
|
const settings = computed(() => store.settings || {
|
||||||
backup_db_nightly_enabled: false,
|
backup_db_nightly_enabled: false,
|
||||||
@@ -345,7 +383,10 @@ function pollOnce() {
|
|||||||
if (offsite.value) {
|
if (offsite.value) {
|
||||||
store.loadRuns('media')
|
store.loadRuns('media')
|
||||||
store.loadRuns('restore')
|
store.loadRuns('restore')
|
||||||
if (polls++ % COVERAGE_EVERY === 0) store.loadMediaCoverage()
|
if (polls++ % COVERAGE_EVERY === 0) {
|
||||||
|
store.loadMediaCoverage()
|
||||||
|
store.loadRuns('reconcile')
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
store.loadRuns('images')
|
store.loadRuns('images')
|
||||||
}
|
}
|
||||||
@@ -353,6 +394,7 @@ function pollOnce() {
|
|||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
store.loadSettings()
|
store.loadSettings()
|
||||||
|
healthStore.refresh()
|
||||||
if (!store.destination) store.loadDestination().catch(() => {})
|
if (!store.destination) store.loadDestination().catch(() => {})
|
||||||
pollOnce()
|
pollOnce()
|
||||||
pollId = setInterval(pollOnce, 5000)
|
pollId = setInterval(pollOnce, 5000)
|
||||||
@@ -367,6 +409,7 @@ async function onTrigger(kind) {
|
|||||||
try {
|
try {
|
||||||
await store.triggerBackup(kind)
|
await store.triggerBackup(kind)
|
||||||
pollOnce()
|
pollOnce()
|
||||||
|
healthStore.refresh()
|
||||||
} finally {
|
} finally {
|
||||||
triggeringRef.value = false
|
triggeringRef.value = false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,8 @@
|
|||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr v-for="r in runs" :key="r.id">
|
<template v-for="r in runs" :key="r.id">
|
||||||
|
<tr>
|
||||||
<td class="fc-tabular" :title="r.started_at">
|
<td class="fc-tabular" :title="r.started_at">
|
||||||
{{ formatRelative(r.started_at) }}
|
{{ formatRelative(r.started_at) }}
|
||||||
</td>
|
</td>
|
||||||
@@ -24,6 +25,12 @@
|
|||||||
<div
|
<div
|
||||||
v-if="r.error" class="fc-run-error text-caption text-error" :title="r.error"
|
v-if="r.error" class="fc-run-error text-caption text-error" :title="r.error"
|
||||||
>{{ r.error }}</div>
|
>{{ r.error }}</div>
|
||||||
|
<v-btn
|
||||||
|
v-if="failures(r).length"
|
||||||
|
size="x-small" variant="text" color="error" class="px-1"
|
||||||
|
:append-icon="open[r.id] ? 'mdi-chevron-up' : 'mdi-chevron-down'"
|
||||||
|
@click="open[r.id] = !open[r.id]"
|
||||||
|
>{{ failedLabel(r) }}</v-btn>
|
||||||
</td>
|
</td>
|
||||||
<td :title="r.offsite ? `${r.offsite.endpoint} · ${r.offsite.key}` : ''">
|
<td :title="r.offsite ? `${r.offsite.endpoint} · ${r.offsite.key}` : ''">
|
||||||
<template v-if="r.offsite">
|
<template v-if="r.offsite">
|
||||||
@@ -57,6 +64,22 @@
|
|||||||
</KebabMenu>
|
</KebabMenu>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<!-- The files a run couldn't send or fetch, each with what went wrong.
|
||||||
|
The run keeps the first few; the count covers them all. -->
|
||||||
|
<tr v-if="open[r.id]" class="fc-failures">
|
||||||
|
<td colspan="7">
|
||||||
|
<ul class="fc-failures__list text-caption">
|
||||||
|
<li v-for="(f, i) in failures(r)" :key="i">
|
||||||
|
<code :title="f.path">{{ f.path }}</code>
|
||||||
|
<span class="text-error"> — {{ f.error }}</span>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<div v-if="moreFailed(r)" class="text-caption fc-muted">
|
||||||
|
and {{ moreFailed(r).toLocaleString() }} more.
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</template>
|
||||||
<tr v-if="!runs.length">
|
<tr v-if="!runs.length">
|
||||||
<td colspan="7" class="text-center fc-muted py-4">{{ emptyText }}</td>
|
<td colspan="7" class="text-center fc-muted py-4">{{ emptyText }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
@@ -65,6 +88,8 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { reactive } from 'vue'
|
||||||
|
|
||||||
import { formatRelative as fmtRelative } from '../../utils/date.js'
|
import { formatRelative as fmtRelative } from '../../utils/date.js'
|
||||||
import KebabMenu from '../common/KebabMenu.vue'
|
import KebabMenu from '../common/KebabMenu.vue'
|
||||||
|
|
||||||
@@ -74,6 +99,23 @@ defineProps({
|
|||||||
})
|
})
|
||||||
defineEmits(['restore', 'delete', 'tag'])
|
defineEmits(['restore', 'delete', 'tag'])
|
||||||
|
|
||||||
|
// Which runs have their failed-file list open.
|
||||||
|
const open = reactive({})
|
||||||
|
|
||||||
|
function failures(r) {
|
||||||
|
return r.manifest?.failures || []
|
||||||
|
}
|
||||||
|
function failedCount(r) {
|
||||||
|
return r.manifest?.failed || failures(r).length
|
||||||
|
}
|
||||||
|
function moreFailed(r) {
|
||||||
|
return Math.max(0, failedCount(r) - failures(r).length)
|
||||||
|
}
|
||||||
|
function failedLabel(r) {
|
||||||
|
const n = failedCount(r)
|
||||||
|
return `${n.toLocaleString()} file${n === 1 ? '' : 's'} failed`
|
||||||
|
}
|
||||||
|
|
||||||
function statusIcon(s) {
|
function statusIcon(s) {
|
||||||
return {
|
return {
|
||||||
ok: 'mdi-check-circle', error: 'mdi-close-circle',
|
ok: 'mdi-check-circle', error: 'mdi-close-circle',
|
||||||
@@ -111,6 +153,12 @@ function formatRelative(iso) {
|
|||||||
<style scoped>
|
<style scoped>
|
||||||
.fc-backup-table { background: transparent; }
|
.fc-backup-table { background: transparent; }
|
||||||
.fc-tabular { font-variant-numeric: tabular-nums; }
|
.fc-tabular { font-variant-numeric: tabular-nums; }
|
||||||
|
.fc-failures td { border-top: 0 !important; }
|
||||||
|
.fc-failures__list { margin: 0 0 6px; padding-left: 18px; }
|
||||||
|
.fc-failures__list code {
|
||||||
|
display: inline-block; max-width: 420px; overflow: hidden;
|
||||||
|
text-overflow: ellipsis; white-space: nowrap; vertical-align: bottom;
|
||||||
|
}
|
||||||
.fc-run-error {
|
.fc-run-error {
|
||||||
max-width: 320px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
max-width: 320px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,12 +10,18 @@ export const useBackupStore = defineStore('backup', () => {
|
|||||||
const imagesRuns = ref([])
|
const imagesRuns = ref([])
|
||||||
const mediaRuns = ref([])
|
const mediaRuns = ref([])
|
||||||
const restoreRuns = ref([])
|
const restoreRuns = ref([])
|
||||||
|
// The weekly check of the bucket against what was sent.
|
||||||
|
const reconcileRuns = ref([])
|
||||||
const settings = ref(null)
|
const settings = ref(null)
|
||||||
const loading = ref({
|
const loading = ref({
|
||||||
dbRuns: false, imagesRuns: false, mediaRuns: false, restoreRuns: false, settings: false,
|
dbRuns: false, imagesRuns: false, mediaRuns: false, restoreRuns: false,
|
||||||
|
reconcileRuns: false, settings: false,
|
||||||
})
|
})
|
||||||
const lastError = ref(null)
|
const lastError = ref(null)
|
||||||
const RUNS = { db: dbRuns, images: imagesRuns, media: mediaRuns, restore: restoreRuns }
|
const RUNS = {
|
||||||
|
db: dbRuns, images: imagesRuns, media: mediaRuns, restore: restoreRuns,
|
||||||
|
reconcile: reconcileRuns,
|
||||||
|
}
|
||||||
|
|
||||||
async function loadRuns(kind) {
|
async function loadRuns(kind) {
|
||||||
const targetRef = RUNS[kind]
|
const targetRef = RUNS[kind]
|
||||||
@@ -170,7 +176,7 @@ export const useBackupStore = defineStore('backup', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
dbRuns, imagesRuns, mediaRuns, restoreRuns, mediaCoverage, loadMediaCoverage,
|
dbRuns, imagesRuns, mediaRuns, restoreRuns, reconcileRuns, mediaCoverage, loadMediaCoverage,
|
||||||
listOffsiteDumps, restoreFromOffsite, fetchMissingFiles,
|
listOffsiteDumps, restoreFromOffsite, fetchMissingFiles,
|
||||||
settings, loading, lastError,
|
settings, loading, lastError,
|
||||||
loadRuns, triggerBackup, restore, deleteRun, setTag,
|
loadRuns, triggerBackup, restore, deleteRun, setTag,
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
|
||||||
|
import BackupRunsTable from '../../src/components/settings/BackupRunsTable.vue'
|
||||||
|
import { mountComponent } from '../support/mountComponent.js'
|
||||||
|
|
||||||
|
const MEDIA_RUN = {
|
||||||
|
id: 7, kind: 'media', status: 'ok', started_at: new Date().toISOString(),
|
||||||
|
duration_seconds: 90, size_bytes: 2048,
|
||||||
|
error: "3 file(s) weren't sent; the next run tries them again.",
|
||||||
|
offsite: { bucket: 'curator', endpoint: 'http://archives.example', key: 'media/' },
|
||||||
|
manifest: {
|
||||||
|
failed: 3,
|
||||||
|
failures: [
|
||||||
|
{ path: '/images/a/one.jpg', error: 'HTTP 503: busy.' },
|
||||||
|
{ path: '/images/b/two.zip', error: 'OSError: gone' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('BackupRunsTable', () => {
|
||||||
|
it('offers a run\'s failed files, counted in full', () => {
|
||||||
|
const w = mountComponent(BackupRunsTable, { props: { runs: [MEDIA_RUN] } })
|
||||||
|
expect(w.text()).toContain('3 files failed')
|
||||||
|
expect(w.text()).not.toContain('/images/a/one.jpg')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('lists each failed file with why once opened, and says how many more', async () => {
|
||||||
|
const w = mountComponent(BackupRunsTable, { props: { runs: [MEDIA_RUN] } })
|
||||||
|
await w.find('v-btn').trigger('click')
|
||||||
|
const t = w.text()
|
||||||
|
expect(t).toContain('/images/a/one.jpg')
|
||||||
|
expect(t).toContain('HTTP 503: busy.')
|
||||||
|
expect(t).toContain('/images/b/two.zip')
|
||||||
|
expect(t).toContain('and 1 more.')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows no failure button on a clean run', () => {
|
||||||
|
const clean = { ...MEDIA_RUN, id: 8, error: null, manifest: { failed: 0, failures: [] } }
|
||||||
|
const w = mountComponent(BackupRunsTable, { props: { runs: [clean] } })
|
||||||
|
expect(w.text()).not.toContain('failed')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""The Backups row on the System view (services/backup_health.py): quiet while
|
||||||
|
the backups keep up, a warning that says what is wrong when the newest dump is
|
||||||
|
more than a day old, the last one failed, or the bucket check found files
|
||||||
|
missing that no run has sent again since."""
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from backend.app.models import BackupRun
|
||||||
|
from backend.app.services import backup_health
|
||||||
|
from backend.app.services.backup_health import BackupFacts, verdict
|
||||||
|
|
||||||
|
NOW = datetime(2026, 3, 4, 12, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_row_when_backups_are_not_set_up():
|
||||||
|
assert verdict(BackupFacts(expected=False, offsite=False), NOW) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_dump_from_last_night_is_ok_and_says_its_age():
|
||||||
|
part = verdict(BackupFacts(expected=True, offsite=True,
|
||||||
|
last_dump_at=NOW - timedelta(hours=9)), NOW)
|
||||||
|
assert part["state"] == "ok"
|
||||||
|
assert part["detail"] == "The newest database backup to the off-site bucket is 9 h old"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_dump_older_than_a_day_warns():
|
||||||
|
part = verdict(BackupFacts(expected=True, offsite=False,
|
||||||
|
last_dump_at=NOW - timedelta(days=3)), NOW)
|
||||||
|
assert part["state"] == "stale"
|
||||||
|
assert part["detail"] == "The newest database backup on this server is 3 days old"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_dump_yet_warns():
|
||||||
|
part = verdict(BackupFacts(expected=True, offsite=True), NOW)
|
||||||
|
assert part["state"] == "stale"
|
||||||
|
assert "No database backup has been taken to the off-site bucket yet" in part["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_failed_dump_warns_with_its_reason_beside_the_age():
|
||||||
|
part = verdict(BackupFacts(expected=True, offsite=True,
|
||||||
|
last_dump_at=NOW - timedelta(hours=30),
|
||||||
|
last_dump_error="S3Error: HTTP 403"), NOW)
|
||||||
|
assert part["state"] == "stale"
|
||||||
|
assert "The last database backup failed: S3Error: HTTP 403" in part["detail"]
|
||||||
|
assert "is 30 h old" in part["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_files_missing_from_the_bucket_warn():
|
||||||
|
part = verdict(BackupFacts(expected=True, offsite=True,
|
||||||
|
last_dump_at=NOW - timedelta(hours=2), missing=1234), NOW)
|
||||||
|
assert part["state"] == "stale"
|
||||||
|
assert "found 1,234 file(s) missing from the bucket" in part["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── reading the facts ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _run(db_sync, kind, status, at, **kw):
|
||||||
|
row = BackupRun(kind=kind, status=status, triggered_by="nightly", started_at=at,
|
||||||
|
finished_at=at + timedelta(minutes=5), **kw)
|
||||||
|
db_sync.add(row)
|
||||||
|
db_sync.commit()
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def offsite(monkeypatch):
|
||||||
|
monkeypatch.setattr("backend.app.services.backup_destination.describe",
|
||||||
|
lambda settings: {"configured": True})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_the_facts_come_from_the_newest_runs(db_sync, offsite):
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
_run(db_sync, "db", "ok", now - timedelta(days=2))
|
||||||
|
newest = _run(db_sync, "db", "ok", now - timedelta(hours=3))
|
||||||
|
_run(db_sync, "db", "running", now - timedelta(minutes=1))
|
||||||
|
|
||||||
|
facts = backup_health.gather(db_sync)
|
||||||
|
assert facts.expected and facts.offsite
|
||||||
|
assert facts.last_dump_at == newest.finished_at
|
||||||
|
assert facts.last_dump_error is None
|
||||||
|
assert facts.missing == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_the_newest_finished_dump_failing_is_reported(db_sync, offsite):
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
_run(db_sync, "db", "ok", now - timedelta(days=2))
|
||||||
|
_run(db_sync, "db", "error", now - timedelta(hours=3), error="DumpFailed: disk full")
|
||||||
|
facts = backup_health.gather(db_sync)
|
||||||
|
assert facts.last_dump_error == "DumpFailed: disk full"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_missing_files_count_until_a_media_run_finishes_after_the_check(db_sync, offsite):
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
check = _run(db_sync, "reconcile", "ok", now - timedelta(hours=5),
|
||||||
|
manifest={"listed": 10, "dropped": 3, "adopted": 0})
|
||||||
|
_run(db_sync, "media", "ok", now - timedelta(days=1))
|
||||||
|
assert backup_health.gather(db_sync).missing == 3
|
||||||
|
|
||||||
|
_run(db_sync, "media", "ok", check.finished_at + timedelta(minutes=1))
|
||||||
|
assert backup_health.gather(db_sync).missing == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_nothing_is_expected_without_a_destination_or_the_nightly(db_sync, monkeypatch):
|
||||||
|
monkeypatch.setattr("backend.app.services.backup_destination.describe",
|
||||||
|
lambda settings: {"configured": False})
|
||||||
|
assert backup_health.gather(db_sync).expected is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
async def test_the_system_view_lists_the_backups_row(client, offsite):
|
||||||
|
body = await (await client.get("/api/system/health")).get_json()
|
||||||
|
part = next(p for p in body["parts"] if p["key"] == "backup")
|
||||||
|
assert part["state"] == "stale"
|
||||||
|
assert part["detail"].startswith("No database backup has been taken")
|
||||||
@@ -324,3 +324,33 @@ def test_reconcile_resends_what_the_bucket_lost_and_adopts_what_it_has(
|
|||||||
"listed": 2, "dropped": 1, "adopted": 1,
|
"listed": 2, "dropped": 1, "adopted": 1,
|
||||||
}
|
}
|
||||||
assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}
|
assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_weekly_check_is_kept_as_a_run_with_its_counts(db_sync, run_env, monkeypatch):
|
||||||
|
from backend.app.tasks import backup as tasks
|
||||||
|
monkeypatch.setattr(backup_media, "reconcile",
|
||||||
|
lambda session, target: {"listed": 9, "dropped": 2, "adopted": 1})
|
||||||
|
|
||||||
|
out = tasks.reconcile_media_backup()
|
||||||
|
row = db_sync.get(BackupRun, out["backup_run_id"])
|
||||||
|
db_sync.refresh(row)
|
||||||
|
assert (row.kind, row.status, row.triggered_by) == ("reconcile", "ok", "weekly")
|
||||||
|
assert row.finished_at is not None
|
||||||
|
assert {k: row.manifest[k] for k in ("listed", "dropped", "adopted")} == {
|
||||||
|
"listed": 9, "dropped": 2, "adopted": 1,
|
||||||
|
}
|
||||||
|
assert row.manifest["offsite"]["bucket"] == "curator"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_weekly_check_that_fails_is_kept_with_why(db_sync, run_env, monkeypatch):
|
||||||
|
from backend.app.tasks import backup as tasks
|
||||||
|
|
||||||
|
def unreachable(session, target):
|
||||||
|
raise S3Error("HTTP 503: busy.")
|
||||||
|
monkeypatch.setattr(backup_media, "reconcile", unreachable)
|
||||||
|
|
||||||
|
with pytest.raises(S3Error):
|
||||||
|
tasks.reconcile_media_backup()
|
||||||
|
row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "reconcile")).scalar_one()
|
||||||
|
assert row.status == "error"
|
||||||
|
assert row.error == "S3Error: HTTP 503: busy."
|
||||||
|
|||||||
Reference in New Issue
Block a user