feat(backup): a backup holds settings and keys only, for recovery (#5822)
#5822 "Backup contents: settings and keys only". The operator's ruling (System 180, 2026-10-15) overturns 2026-10-12's "Config + audit + alerts": a backup is for system recovery and carries what would otherwise have to be set up again, with no logs or history. - contents.py: audit_events, alert_events, alert_states, metric_anomalies, metric_anomaly_spans and user_invites move to EXCLUDED. - Alert state is no longer restored, and evaluation skips a rule with no state row. ensure_rule_states (alerts/service.py) gives every single-resource rule lacking one an inactive row; alert_reconcile runs it each minute. - Docs, the Backups page and the CLI help say what a backup holds now. CI's image check also finds the audit log empty after a restore. - tests/integration/test_backup_restore_fixes.py is removed: its case (an alert event naming a left-out Ansible run) no longer arises. The SET NULL fix-up is covered by a unit test on synthetic tables. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -744,6 +744,25 @@ async def reconcile_covering_rules(app) -> None:
|
||||
", ".join(added) or "none", ", ".join(removed) or "none")
|
||||
|
||||
|
||||
async def ensure_rule_states(app) -> int:
|
||||
"""A state row, inactive, for every rule watching one resource that has
|
||||
none. Evaluation skips a rule without one, and a backup leaves alert state
|
||||
out (System 180, 2026-10-15), so a restored rule comes back without it.
|
||||
Run each minute by the core scheduler (alert_reconcile): a restored rule is
|
||||
watched again within a minute. Returns how many rows were added."""
|
||||
async with app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
has_state = select(AlertState.id).where(AlertState.rule_id == AlertRule.id).exists()
|
||||
missing = (await db.execute(select(AlertRule.id).where(
|
||||
AlertRule.covers == "resource", ~has_state))).scalars().all()
|
||||
now = datetime.now(timezone.utc)
|
||||
db.add_all([AlertState(rule_id=rule_id, state=AlertStateEnum.inactive,
|
||||
last_transition_at=now) for rule_id in missing])
|
||||
if missing:
|
||||
logger.info("Alert rules without a state row: added %d", len(missing))
|
||||
return len(missing)
|
||||
|
||||
|
||||
def part_of(rule: AlertRule, rules) -> AlertRule | None:
|
||||
"""The covering rule a child belongs to, from a list of rules; None for a
|
||||
rule that stands alone."""
|
||||
|
||||
+3
-2
@@ -429,9 +429,10 @@ def _register_core_tasks(app: Quart) -> None:
|
||||
await record_cleanup_secs(app, done["secs"])
|
||||
|
||||
async def run_alert_reconcile():
|
||||
from .alerts.service import reconcile_covering_rules
|
||||
from .alerts.service import ensure_rule_states, reconcile_covering_rules
|
||||
from .core.alerts import check_silence, notify_after_maintenance
|
||||
for step in (reconcile_covering_rules, check_silence, notify_after_maintenance):
|
||||
for step in (reconcile_covering_rules, ensure_rule_states, check_silence,
|
||||
notify_after_maintenance):
|
||||
try:
|
||||
await step(app)
|
||||
except Exception:
|
||||
|
||||
+18
-13
@@ -2,38 +2,43 @@
|
||||
or left out, and tests/core/test_backup_contents.py fails on one that is
|
||||
neither, so a new table is a decision someone makes in this file.
|
||||
|
||||
The operator's ruling (System 180 "Backup & Restore", 2026-10-12): a backup
|
||||
carries configuration, the audit log and alert history. Metrics, check
|
||||
results and the Docker, Traefik and UniFi history are left out: their tables
|
||||
come back with their structure and no rows, and fill again as Steward runs.
|
||||
The operator's ruling (System 180 "Backup & Restore", 2026-10-15): a backup
|
||||
is for system recovery. It carries settings and keys, what would otherwise
|
||||
have to be set up again, and no logs or history: not the audit log, not
|
||||
alert history, not metrics. Left-out tables come back with their structure
|
||||
and no rows, and fill again as Steward runs.
|
||||
|
||||
A table that is in neither list (an external plugin's) is backed up whole:
|
||||
a table nobody thought about is kept, never lost.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# Configuration, and the history the operator chose to keep.
|
||||
# What would otherwise have to be set up again.
|
||||
KEPT: frozenset[str] = frozenset({
|
||||
"app_settings", "users", "user_invites", "api_tokens",
|
||||
"app_settings", "users", "api_tokens",
|
||||
"hosts", "host_agent_registrations",
|
||||
# A group's maintenance cadence (#5768): configuration.
|
||||
"host_maintenance_plans",
|
||||
"monitors",
|
||||
"alert_rules", "alert_states", "maintenance_windows",
|
||||
"alert_rules", "maintenance_windows",
|
||||
"ansible_targets", "ansible_groups", "ansible_target_groups", "ansible_schedules",
|
||||
"dashboards", "dashboard_widgets", "dashboard_share_tokens",
|
||||
"snmp_devices", "prometheus_targets", "prometheus_widget_templates",
|
||||
# Names given to network clients by hand (UniFi's alias column).
|
||||
"unifi_known_clients",
|
||||
# History kept by the ruling.
|
||||
"audit_events", "alert_events",
|
||||
# Metric anomalies and their reviews (#5740): what someone marked normal
|
||||
# or real is a decision, kept like alert history.
|
||||
"metric_anomalies", "metric_anomaly_spans",
|
||||
})
|
||||
|
||||
# Rows that are measurements, logs or in-flight work: left out.
|
||||
# Rows that are measurements, logs, history or in-flight work: left out.
|
||||
EXCLUDED: frozenset[str] = frozenset({
|
||||
# Logs and history (the 2026-10-15 ruling): a recovery doesn't need them.
|
||||
"audit_events", "alert_events",
|
||||
# Live state: each rule starts inactive, and alert_reconcile gives a
|
||||
# restored rule its state row within a minute (ensure_rule_states).
|
||||
"alert_states",
|
||||
# Metric anomalies and their reviews (#5740): history of the metrics.
|
||||
"metric_anomalies", "metric_anomaly_spans",
|
||||
# A pending invite is in flight, not setup; it can be sent again.
|
||||
"user_invites",
|
||||
"plugin_metrics", "plugin_metric_latest", "plugin_metrics_hourly",
|
||||
# Learned from the metrics above, so relearned once they fill again.
|
||||
"metric_norms",
|
||||
|
||||
+2
-2
@@ -65,8 +65,8 @@ def main(ctx: click.Context, config: str, host: str, port: int, debug: bool) ->
|
||||
help="The folder to write the backup file into.")
|
||||
@click.pass_context
|
||||
def backup_command(ctx: click.Context, out_dir: Path) -> None:
|
||||
"""Write a backup file of this Steward (configuration, audit log, alert
|
||||
history and the secret key) into a folder, without sending it anywhere."""
|
||||
"""Write a backup file of this Steward (settings and the secret key: what
|
||||
recovery needs, no logs or history) into a folder, without sending it anywhere."""
|
||||
from .backup.dump import BackupError, make_backup
|
||||
from .backup.job import migration_heads
|
||||
from .config import load_bootstrap
|
||||
|
||||
@@ -352,7 +352,8 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
|
||||
for rule in rules:
|
||||
alert_state = states.get(rule.id)
|
||||
if alert_state is None:
|
||||
logger.warning("No alert_state row for rule %s, skipping", rule.id)
|
||||
# A restored rule (backups leave state out); alert_reconcile adds it.
|
||||
logger.warning("No alert_state row for rule %s yet, skipping", rule.id)
|
||||
continue
|
||||
samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
|
||||
key=lambda r: r.recorded_at)
|
||||
|
||||
@@ -30,8 +30,9 @@
|
||||
<div class="card" style="margin-bottom:1.25rem;">
|
||||
<h2 class="section-title" style="margin-bottom:0.5rem;">Bucket</h2>
|
||||
<p class="field-hint" style="margin-bottom:1rem;">
|
||||
Each backup is one file, <code>steward-<time>.tar</code>: Steward's configuration, its audit log
|
||||
and alert history, and its secret key. Metrics and check results are left out. Anyone holding a
|
||||
Each backup is one file, <code>steward-<time>.tar</code>, for getting Steward running again:
|
||||
its settings, users, hosts, monitors, alert rules and dashboards, and its secret key. Logs and
|
||||
history are left out: the audit log, alert history, metrics and check results. Anyone holding a
|
||||
copy can read every password and key Steward keeps, so keep the bucket private.
|
||||
</p>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user