bvandeusenandClaude Opus 5.5
c368e93e2c
Agent offline is an ordinary rule now. The host_agent scheduler records host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or registration time for an agent that never reported), replacing the task that only logged stale agents. The catalog offers the metric, and picking it on a new rule fills "> 180". No data: each rule's state tracks its last sample and its usual gap, smoothed so that an outage counts as at most three gaps. A core task, alert_silence_check, runs every 60s and marks a rule no_data_since once its series has had no sample for three usual gaps (at least 5 minutes; an hour while the cadence is unknown). A never-sampled rule counts from when it started watching. The mark sits beside the state, so a firing alert that goes silent stays firing. Each silence records one event and sends one notification (rule option notify_no_data, default on; muted in maintenance windows). The next sample clears it with a "data resumed" event and notification. Creating, changing, disabling or re-enabling a rule restarts its watch. Migration 0029 adds the columns and seeds each existing rule's last sample and gap from its last day of data, so an upgrade does not flag every rule. Shown on the Alerts page (a NO DATA badge with its start time), in MCP list_alerts, list_alert_rules and get_host (state "no_data", no_data_since), in get_overview findings (alert_no_data, or a firing alert marked silent) and in the weekly report. The rule form has the notify option, and its metric reference now matches what traefik records and lists host_agent. Emails say "no data" and carry the event's note. The metric-catalog guard also reads positional MetricBatch.add arguments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
131 lines
6.5 KiB
Python
131 lines
6.5 KiB
Python
from __future__ import annotations
|
|
import enum
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from sqlalchemy import JSON, Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, Text
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
from .base import Base
|
|
|
|
|
|
class AlertOperator(str, enum.Enum):
|
|
gt = ">"
|
|
lt = "<"
|
|
gte = ">="
|
|
lte = "<="
|
|
eq = "=="
|
|
ne = "!="
|
|
|
|
|
|
class AlertStateEnum(str, enum.Enum):
|
|
inactive = "inactive"
|
|
pending = "pending"
|
|
firing = "firing"
|
|
acknowledged = "acknowledged"
|
|
resolved = "resolved"
|
|
|
|
|
|
class AlertRule(Base):
|
|
__tablename__ = "alert_rules"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
|
name: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
# The DB enum (migration 0002) stores the operator VALUES (">", "<", …), so
|
|
# persist values rather than SQLAlchemy's default enum-member NAMES ("gt"…).
|
|
operator: Mapped[AlertOperator] = mapped_column(
|
|
Enum(AlertOperator, name="alertoperator",
|
|
values_callable=lambda e: [m.value for m in e]),
|
|
nullable=False,
|
|
)
|
|
threshold: Mapped[float] = mapped_column(Float, nullable=False)
|
|
consecutive_failures_required: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
|
# Non-breaching checks in a row before a firing (or acknowledged) alert resolves.
|
|
consecutive_ok_required: Mapped[int] = mapped_column(Integer, nullable=False, default=1,
|
|
server_default="1")
|
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
# Notify when the watched series goes silent (no_data) and when it resumes.
|
|
notify_no_data: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True,
|
|
server_default="true")
|
|
# Optional admin-only action: run an Ansible playbook on transition-to-firing.
|
|
# Shape: {source, playbook, inventory, extra_vars, limit, tags, check}. NULL = none.
|
|
ansible_action: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
|
created_by: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="RESTRICT"), nullable=False)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
|
)
|
|
|
|
|
|
class AlertState(Base):
|
|
__tablename__ = "alert_states"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
|
rule_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("alert_rules.id", ondelete="CASCADE"), nullable=False, unique=True
|
|
)
|
|
state: Mapped[AlertStateEnum] = mapped_column(
|
|
Enum(AlertStateEnum), nullable=False, default=AlertStateEnum.inactive
|
|
)
|
|
consecutive_failure_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
consecutive_ok_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0,
|
|
server_default="0")
|
|
last_transition_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
|
)
|
|
acknowledged_by: Mapped[str | None] = mapped_column(
|
|
String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
|
)
|
|
acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
# Silence tracking (core.alerts.check_silence): when the watched series last
|
|
# had a sample, its usual gap between samples, and since when it has had
|
|
# none for too long. no_data_since is a mark beside the state, not a state:
|
|
# a firing alert that goes silent stays firing.
|
|
last_sample_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
sample_interval_s: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
no_data_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
|
|
|
|
class MaintenanceWindow(Base):
|
|
__tablename__ = "maintenance_windows"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
|
name: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
start_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
end_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
# Scope — null means "all". scope_resource requires scope_source to be set.
|
|
scope_source: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
scope_resource: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
created_by: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("users.id", ondelete="RESTRICT"), nullable=False
|
|
)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
|
)
|
|
|
|
|
|
class AlertEvent(Base):
|
|
__tablename__ = "alert_events"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
|
rule_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("alert_rules.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
from_state: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
to_state: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
# None when the transition came from a change to the rule, not a sample.
|
|
metric_value: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
# Why a transition happened when the value alone doesn't say (a rule edit).
|
|
note: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
transitioned_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
|
)
|
|
email_sent: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
email_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
webhook_sent: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
webhook_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
# Set when this firing event triggered an Ansible action run (audit link).
|
|
ansible_run_id: Mapped[str | None] = mapped_column(
|
|
String(36), ForeignKey("ansible_runs.id", ondelete="SET NULL"), nullable=True
|
|
)
|