Files
FabledSteward/steward/audit/service.py
T
bvandeusenandClaude Opus 5.5 5569f9934e
CI / lint (push) Successful in 2s
CI / integration (push) Failing after 1m24s
CI / unit (push) Successful in 1m37s
CI / publish (push) Skipped
feat(anomalies): a metric out of its normal is an anomaly to review (#5740)
#5740 "A metric that departs from its normal is recorded as an anomaly to
review, and is findable after it ends", milestone 513.

- Every minute each learned key's last 15 minutes is judged against its
  hour's normal: above the usual hold x1.25 and by the metric's floor opens
  an occurrence, which ends when the level comes back.
- metric_anomalies holds one row per item to review; a repeat while it
  still waits joins it as an occurrence (metric_anomaly_spans).
- An ended row nobody reviews within anomalies.review_days (14, Settings ->
  Thresholds) lapses; an ongoing one never does.
- Learning skips every hour an occurrence covers unless its anomaly was
  marked normal: no normal is learned without review.
- Alerts -> Anomalies lists those to review first, then the reviewed and
  lapsed under day dividers; marking is operator/admin and audited.
- The Overview widget and get_overview warn of open ones and count those
  to review; the host page shows its own; the weekly report lists the
  week's and the lapsed; MCP list_anomalies and review_anomaly.
- Reviewed and lapsed rows are pruned at data.retention_days, and kept
  in backups like alert history.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 13:41:57 -04:00

280 lines
13 KiB
Python

"""Name the thing each audit entry is about.
An audit row stores an entity type and id; the page shows the entity's
current name, linked to its page, in one query per type for the whole page.
An entity deleted since keeps whatever name the entry's own detail recorded.
"""
from __future__ import annotations
import logging
from sqlalchemy import select
from steward.ansible.inventory_gen import run_title, scope_labels
from steward.core.formatting import day_group
from steward.models.alerts import AlertRule, MaintenanceWindow
from steward.models.ansible import AnsibleRun
from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
from steward.models.ansible_schedule import AnsibleSchedule
from steward.models.api_tokens import ApiToken
from steward.models.hosts import Host
from steward.models.monitors import Monitor
from steward.models.users import User
logger = logging.getLogger(__name__)
# entity_type -> (model, its page for an id). A type named by a value rather
# than a row (plugin, ansible_playbook) is its own label.
_NAMED = {
"host": (Host, "/hosts/{}"),
"monitor": (Monitor, "/monitors/{}"),
"alert_rule": (AlertRule, "/alerts/history?rule={}"),
"maintenance_window": (MaintenanceWindow, "/alerts/maintenance"),
"ansible_target": (AnsibleTarget, "/ansible/inventory/targets/{}"),
"ansible_group": (AnsibleGroup, "/ansible/inventory/groups/{}"),
"ansible_schedule": (AnsibleSchedule, "/ansible/schedules"),
"api_token": (ApiToken, "/ai-access/"),
"user": (User, "/settings/users/{}"),
}
_SELF_NAMED = {"plugin": "/settings/plugins/", "ansible_playbook": None}
def _named_types() -> dict:
"""Core's named entity types plus those a loaded plugin names through its
get_audit_entities() hook ({type: (model, page for an id)}), so core
reads a plugin's rows without importing it."""
from steward.core.plugin_manager import get_loaded_plugin_modules
named = dict(_NAMED)
for name, module in get_loaded_plugin_modules().items():
hook = getattr(module, "get_audit_entities", None)
if hook is None:
continue
try:
named.update(hook() or {})
except Exception:
logger.exception("Plugin %r: get_audit_entities() raised; its entries show by id", name)
return named
def _recorded_name(detail: dict) -> str | None:
"""A name the entry recorded when it was written, for a deleted entity."""
for key in ("name", "new_name", "username", "playbook"):
if detail.get(key):
return str(detail[key])
return None
async def entity_labels(db, items: list[dict]) -> dict[tuple[str, str], dict]:
"""{(entity_type, entity_id): {"label", "url", "gone"}} for the audit rows
in `items` ({"event", "detail"}), batched one query per type."""
ids: dict[str, set[str]] = {}
for it in items:
e = it["event"]
if e.entity_type and e.entity_id:
ids.setdefault(e.entity_type, set()).add(str(e.entity_id))
named = _named_types()
labels: dict[tuple[str, str], dict] = {}
for etype, wanted in ids.items():
if etype in named:
model, page = named[etype]
key_col = model.id
keys = [int(i) for i in wanted if i.isdigit()] if etype == "api_token" else list(wanted)
if not keys:
continue
name_col = model.username if model is User else model.name
for row_id, name in (await db.execute(
select(key_col, name_col).where(key_col.in_(keys)))).all():
labels[(etype, str(row_id))] = {"label": name, "url": page.format(row_id), "gone": False}
elif etype == "ansible_run":
runs = (await db.execute(
select(AnsibleRun).where(AnsibleRun.id.in_(list(wanted))))).scalars().all()
scopes = await scope_labels(db, [r.inventory_scope for r in runs])
for r in runs:
labels[(etype, r.id)] = {
"label": run_title(r.playbook_path, scopes.get(r.inventory_scope, r.inventory_scope)),
"url": f"/ansible/runs/{r.id}", "gone": False}
elif etype in _SELF_NAMED:
for i in wanted:
labels[(etype, i)] = {"label": i, "url": _SELF_NAMED[etype], "gone": False}
# What a lookup did not find was deleted since: keep the name the entry recorded.
for it in items:
e = it["event"]
if not (e.entity_type and e.entity_id):
continue
key = (e.entity_type, str(e.entity_id))
if key not in labels:
recorded = _recorded_name(it["detail"] or {})
labels[key] = {"label": recorded or f"{str(e.entity_id)[:8]}", "url": None, "gone": True}
return labels
# What each action did, said as the rest of a sentence that starts with who
# did it: "admin paused monitor api check". An action not listed here is
# shown by its code, which is also every row's tooltip and what the filter
# matches.
ACTIONS = {
"auth.login": "signed in",
"auth.login_failed": "failed to sign in",
"auth.setup": "set up Steward",
"account.time_zone": "set their time zone",
"account.email": "changed their email settings",
"account.password_changed": "changed their password",
"user.password_reset_requested": "asked for a password reset email",
"user.password_reset_by_email": "reset their password from an emailed link",
"user.created": "added user",
"user.renamed": "renamed user",
"user.role_changed": "changed the role of user",
"user.email_settings": "changed the email settings of user",
"user.password_reset": "set a new password for user",
"user.disabled": "disabled user",
"user.enabled": "enabled user",
"user.deleted": "deleted user",
"user.invited": "invited",
"user.invite_resent": "resent the invite to",
"user.invite_revoked": "withdrew the invite to",
"user.joined": "joined from an invite as",
"host.created": "added host",
"host.renamed": "renamed host",
"host.deleted": "deleted host",
"monitor.created": "added monitor",
"monitor.updated": "changed monitor",
"monitor.enabled": "resumed monitor",
"monitor.disabled": "paused monitor",
"monitor.deleted": "deleted monitor",
"alert_rule.created": "added alert rule",
"alert_rule.updated": "changed alert rule",
"alert_rule.enabled": "turned on alert rule",
"alert_rule.disabled": "turned off alert rule",
"alert_rule.deleted": "deleted alert rule",
"alert_rule.acknowledged": "acknowledged the alert from",
"alert_rule.tested": "sent a test alert for rule",
"alert_rule.action_set": "set the playbook run by alert rule",
"alert_rule.action_cleared": "removed the playbook from alert rule",
"maintenance_window.created": "scheduled maintenance",
"maintenance_window.deleted": "deleted maintenance window",
"maintenance_window.ended": "ended maintenance window",
"maintenance_window.cancelled": "cancelled maintenance window",
"ansible.run.started": "started",
"ansible.run.cancelled": "cancelled",
"ansible.host_run": "ran a playbook on host",
"ansible.schedule.created": "added schedule",
"ansible.schedule.updated": "changed schedule",
"ansible.schedule.enabled": "resumed schedule",
"ansible.schedule.disabled": "paused schedule",
"ansible.schedule.deleted": "deleted schedule",
"ansible.schedule.run_now": "ran schedule now",
"ansible.playbook.created": "added playbook",
"ansible.playbook.updated": "edited playbook",
"ansible.playbook.deleted": "deleted playbook",
"ansible.target.created": "added inventory target",
"ansible.target.updated": "changed inventory target",
"ansible.target.deleted": "deleted inventory target",
"ansible.group.created": "added inventory group",
"ansible.group.updated": "changed inventory group",
"ansible.group.deleted": "deleted inventory group",
"settings.saved": "saved settings",
"settings.notification.tested": "sent a test notification",
"plugin.enabled": "turned on plugin",
"plugin.disabled": "turned off plugin",
"plugin.install": "installed plugin",
"plugin.install_failed": "failed to install plugin",
"plugin.settings_removed": "removed the settings of plugin",
"api_token.create": "created AI access token",
"api_token.revoke": "revoked AI access token",
"docker.prune.started": "started a Docker cleanup on host",
"snmp.device_created": "added SNMP device",
"snmp.device_updated": "changed SNMP device",
"snmp.device_deleted": "deleted SNMP device",
"prometheus.target_created": "added Prometheus target",
"prometheus.target_updated": "changed Prometheus target",
"prometheus.target_deleted": "deleted Prometheus target",
"prometheus.template_created": "added Prometheus widget template",
"prometheus.template_updated": "changed Prometheus widget template",
"prometheus.template_deleted": "deleted Prometheus widget template",
"backup.made": "sent a backup",
"backup.failed": "tried to send a backup, which failed:",
}
# The filter's choices: an area, matched as a prefix of the action code.
AREAS = (("auth.", "Sign-ins"), ("account.", "Accounts"), ("user.", "Users"), ("host.", "Hosts"), ("monitor.", "Monitors"),
("alert_rule.", "Alert rules"), ("maintenance_window.", "Maintenance"),
("ansible.run", "Playbook runs"), ("ansible.", "Ansible (all)"),
("settings.", "Settings"), ("plugin.", "Plugins"), ("api_token.", "AI access"),
("docker.", "Docker"), ("snmp.", "SNMP"), ("prometheus.", "Prometheus"),
("backup.", "Backups"))
def action_text(action: str) -> str:
return ACTIONS.get(action, action)
# Detail keys a row does not need to repeat: the entity column already names
# and links the thing, and the scope's own wording is shown in its place.
_SKIP = {"rule_id", "monitor_id", "host_id", "window_id", "schedule_id", "scope_label"}
_LABELS = {"check": "dry run", "interval_seconds": "every", "interval_s": "every",
"bytes": "size", "renamed_from": "was", "token_id": "AI token", "vars": "variables",
"extra_vars": "variables", "secret_vars": "secret variables", "sent": "delivered",
"what": "removing", "address": "address", "carried": "history carried over"}
_METHODS = {"local": "password", "ldap": "LDAP", "oidc": "single sign-on"}
_SECTIONS = {"general": "General", "thresholds": "Thresholds", "notifications": "Notifications",
"ansible.managed_key": "Ansible: managed SSH key",
"ansible.credentials": "Ansible: credentials", "auth": "Authentication",
"backups": "Backups"}
def _plain(value) -> str:
if isinstance(value, bool):
return "yes" if value else "no"
if isinstance(value, (list, tuple)):
return ", ".join(_plain(v) for v in value) if value else "none"
if isinstance(value, dict):
return ", ".join(f"{str(k).replace('_', ' ')} {_plain(v)}" for k, v in value.items()) or "none"
return str(value)
def detail_facts(detail: dict, scopes: dict[str, str]) -> list[dict]:
"""An entry's detail as [{"label", "text", "url"}] a reader can take in:
yes/no for flags, lists joined, a run's scope in words (scope_labels),
a span as a span, and nothing the entity column already says."""
from steward.core.formatting import human_bytes, human_duration
facts = []
for key, value in (detail or {}).items():
if key in _SKIP or value is None:
continue
label, url = _LABELS.get(key, key.replace("_", " ")), None
if key == "scope":
text = scopes.get(value, value)
elif key in ("interval_seconds", "interval_s"):
text = human_duration(value)
elif key == "bytes":
text = human_bytes(value)
elif key == "method":
text = _METHODS.get(value, value)
elif key == "section":
text = _SECTIONS.get(value, value)
elif key == "token_id":
text = f"#{value}"
elif key == "run_id":
label, text, url = "run", "open", f"/ansible/runs/{value}"
else:
text = _plain(value)
facts.append({"label": label, "text": text, "url": url})
return facts
def mark_day_groups(items: list[dict], tz, group: str = "", now=None,
at=lambda item: item["event"].timestamp) -> str:
"""Give each newest-first entry its recency group (formatting.day_group,
in the reader's zone) and whether it opens one, so the page draws a
divider there. `group` is the one the previous page ended in: a group
split across pages is opened once. `at` reads an entry's moment.
Returns the group this page ends in."""
for item in items:
item["group"] = day_group(at(item), tz, now)
item["opens_group"] = item["group"] != group
group = item["group"]
return group