feat(backup): a backup holds settings and keys only, for recovery (#5822)
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 1m25s
CI / unit (push) Successful in 1m36s
CI / publish (push) Successful in 1m10s

#5822 "Backup contents: settings and keys only". The operator's ruling
(System 180, 2026-10-15) overturns 2026-10-12's "Config + audit + alerts":
a backup is for system recovery and carries what would otherwise have to
be set up again, with no logs or history.

- contents.py: audit_events, alert_events, alert_states, metric_anomalies,
  metric_anomaly_spans and user_invites move to EXCLUDED.
- Alert state is no longer restored, and evaluation skips a rule with no
  state row. ensure_rule_states (alerts/service.py) gives every
  single-resource rule lacking one an inactive row; alert_reconcile runs
  it each minute.
- Docs, the Backups page and the CLI help say what a backup holds now.
  CI's image check also finds the audit log empty after a restore.
- tests/integration/test_backup_restore_fixes.py is removed: its case (an
  alert event naming a left-out Ansible run) no longer arises. The SET NULL
  fix-up is covered by a unit test on synthetic tables.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-15 10:39:55 -04:00
co-authored by Claude Opus 5.5
parent b29b160baa
commit e6da0ff299
11 changed files with 147 additions and 99 deletions
+19
View File
@@ -744,6 +744,25 @@ async def reconcile_covering_rules(app) -> None:
", ".join(added) or "none", ", ".join(removed) or "none")
async def ensure_rule_states(app) -> int:
"""A state row, inactive, for every rule watching one resource that has
none. Evaluation skips a rule without one, and a backup leaves alert state
out (System 180, 2026-10-15), so a restored rule comes back without it.
Run each minute by the core scheduler (alert_reconcile): a restored rule is
watched again within a minute. Returns how many rows were added."""
async with app.db_sessionmaker() as db:
async with db.begin():
has_state = select(AlertState.id).where(AlertState.rule_id == AlertRule.id).exists()
missing = (await db.execute(select(AlertRule.id).where(
AlertRule.covers == "resource", ~has_state))).scalars().all()
now = datetime.now(timezone.utc)
db.add_all([AlertState(rule_id=rule_id, state=AlertStateEnum.inactive,
last_transition_at=now) for rule_id in missing])
if missing:
logger.info("Alert rules without a state row: added %d", len(missing))
return len(missing)
def part_of(rule: AlertRule, rules) -> AlertRule | None:
"""The covering rule a child belongs to, from a list of rules; None for a
rule that stands alone."""
+3 -2
View File
@@ -429,9 +429,10 @@ def _register_core_tasks(app: Quart) -> None:
await record_cleanup_secs(app, done["secs"])
async def run_alert_reconcile():
from .alerts.service import reconcile_covering_rules
from .alerts.service import ensure_rule_states, reconcile_covering_rules
from .core.alerts import check_silence, notify_after_maintenance
for step in (reconcile_covering_rules, check_silence, notify_after_maintenance):
for step in (reconcile_covering_rules, ensure_rule_states, check_silence,
notify_after_maintenance):
try:
await step(app)
except Exception:
+18 -13
View File
@@ -2,38 +2,43 @@
or left out, and tests/core/test_backup_contents.py fails on one that is
neither, so a new table is a decision someone makes in this file.
The operator's ruling (System 180 "Backup & Restore", 2026-10-12): a backup
carries configuration, the audit log and alert history. Metrics, check
results and the Docker, Traefik and UniFi history are left out: their tables
come back with their structure and no rows, and fill again as Steward runs.
The operator's ruling (System 180 "Backup & Restore", 2026-10-15): a backup
is for system recovery. It carries settings and keys, what would otherwise
have to be set up again, and no logs or history: not the audit log, not
alert history, not metrics. Left-out tables come back with their structure
and no rows, and fill again as Steward runs.
A table that is in neither list (an external plugin's) is backed up whole:
a table nobody thought about is kept, never lost.
"""
from __future__ import annotations
# Configuration, and the history the operator chose to keep.
# What would otherwise have to be set up again.
KEPT: frozenset[str] = frozenset({
"app_settings", "users", "user_invites", "api_tokens",
"app_settings", "users", "api_tokens",
"hosts", "host_agent_registrations",
# A group's maintenance cadence (#5768): configuration.
"host_maintenance_plans",
"monitors",
"alert_rules", "alert_states", "maintenance_windows",
"alert_rules", "maintenance_windows",
"ansible_targets", "ansible_groups", "ansible_target_groups", "ansible_schedules",
"dashboards", "dashboard_widgets", "dashboard_share_tokens",
"snmp_devices", "prometheus_targets", "prometheus_widget_templates",
# Names given to network clients by hand (UniFi's alias column).
"unifi_known_clients",
# History kept by the ruling.
"audit_events", "alert_events",
# Metric anomalies and their reviews (#5740): what someone marked normal
# or real is a decision, kept like alert history.
"metric_anomalies", "metric_anomaly_spans",
})
# Rows that are measurements, logs or in-flight work: left out.
# Rows that are measurements, logs, history or in-flight work: left out.
EXCLUDED: frozenset[str] = frozenset({
# Logs and history (the 2026-10-15 ruling): a recovery doesn't need them.
"audit_events", "alert_events",
# Live state: each rule starts inactive, and alert_reconcile gives a
# restored rule its state row within a minute (ensure_rule_states).
"alert_states",
# Metric anomalies and their reviews (#5740): history of the metrics.
"metric_anomalies", "metric_anomaly_spans",
# A pending invite is in flight, not setup; it can be sent again.
"user_invites",
"plugin_metrics", "plugin_metric_latest", "plugin_metrics_hourly",
# Learned from the metrics above, so relearned once they fill again.
"metric_norms",
+2 -2
View File
@@ -65,8 +65,8 @@ def main(ctx: click.Context, config: str, host: str, port: int, debug: bool) ->
help="The folder to write the backup file into.")
@click.pass_context
def backup_command(ctx: click.Context, out_dir: Path) -> None:
"""Write a backup file of this Steward (configuration, audit log, alert
history and the secret key) into a folder, without sending it anywhere."""
"""Write a backup file of this Steward (settings and the secret key: what
recovery needs, no logs or history) into a folder, without sending it anywhere."""
from .backup.dump import BackupError, make_backup
from .backup.job import migration_heads
from .config import load_bootstrap
+2 -1
View File
@@ -352,7 +352,8 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
for rule in rules:
alert_state = states.get(rule.id)
if alert_state is None:
logger.warning("No alert_state row for rule %s, skipping", rule.id)
# A restored rule (backups leave state out); alert_reconcile adds it.
logger.warning("No alert_state row for rule %s yet, skipping", rule.id)
continue
samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
key=lambda r: r.recorded_at)
+3 -2
View File
@@ -30,8 +30,9 @@
<div class="card" style="margin-bottom:1.25rem;">
<h2 class="section-title" style="margin-bottom:0.5rem;">Bucket</h2>
<p class="field-hint" style="margin-bottom:1rem;">
Each backup is one file, <code>steward-&lt;time&gt;.tar</code>: Steward's configuration, its audit log
and alert history, and its secret key. Metrics and check results are left out. Anyone holding a
Each backup is one file, <code>steward-&lt;time&gt;.tar</code>, for getting Steward running again:
its settings, users, hosts, monitors, alert rules and dashboards, and its secret key. Logs and
history are left out: the audit log, alert history, metrics and check results. Anyone holding a
copy can read every password and key Steward keeps, so keep the bucket private.
</p>