Compare commits
4
Commits
a24b9da69d
...
22da64a284
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
22da64a284 | ||
|
|
398312b8be | ||
|
|
622addc09e | ||
|
|
91bb199d14 |
+3
-1
@@ -159,6 +159,8 @@ exposes `register_mcp_tools(register)`, optionally `mcp_overview(db)` to
|
||||
contribute findings to `get_overview`, and optionally
|
||||
`mcp_events(db, host_id, host_name, since, until)`: what it knows happened on a
|
||||
host in a window, as a list of `steward.mcp.tools._events.event(...)`, which
|
||||
`get_anomaly` shows beside an anomaly. See
|
||||
`get_anomaly` shows beside an anomaly; for a resource with no host, optionally
|
||||
`mcp_resource_events(db, source, resource, since, until)`. Core adds the
|
||||
stretches Steward itself wasn't recording to every anomaly. See
|
||||
[Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
|
||||
anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.
|
||||
|
||||
@@ -103,7 +103,7 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r
|
||||
- **Slow responses** compare p95 with p50. When p95 is 4× p50 or more, a few requests are slow (one endpoint or a heavy page). When p50 is close behind, the whole app is slow.
|
||||
- **Next:** a fault links to the swarm service behind the router (its tasks and logs) when Steward can match the name ("codex@swarm" → `codex_app`, "radarr@swarm" → `arr_radarr`), otherwise to Docker → Swarm. Traffic links to Settings → Thresholds.
|
||||
4. **Traefik's own figures:** when its config last reloaded, open connections, memory.
|
||||
5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on.
|
||||
5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on. A certificate Traefik has renewed is listed as old, not by its expiry, though a Traefik that hasn't restarted since keeps reporting it: one whose every name a later-expiring certificate covers.
|
||||
|
||||
A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range.
|
||||
|
||||
|
||||
@@ -403,9 +403,13 @@ async def mcp_events(db, host_id, host_name, since, until) -> list[dict]:
|
||||
for r in await _jobs_on(db, host_id, since, until)]
|
||||
```
|
||||
|
||||
- Return what happened between `since` and `until`, oldest first: something that lasted gives `until`. Keep it to tens of rows; the caller keeps the first 40.
|
||||
- Return what happened between `since` and `until`, oldest first: something that lasted gives `until`. Bound your query (a few hundred rows); Steward keeps the 40 nearest the occurrence's start and tells the caller how many it left out (`around_left_out`).
|
||||
- It is called only while the plugin is enabled, with its own session. If it raises, the anomaly is still returned without the plugin's events. The host agent (maintenance jobs, reboots) and Docker (container starts, deaths, OOM kills) are the examples to read.
|
||||
|
||||
A resource with no host (a Traefik router, say) has no host to read events by. A plugin that knows what happened to one of its own resources exports `mcp_resource_events(db, source, resource, since, until)` instead, same return shape, and returns `[]` for a source that isn't its own. Traefik's is the example: it gives a router's swarm service's container events, on whichever hosts they ran.
|
||||
|
||||
Core adds, beside every anomaly, the stretches Steward itself wasn't recording (kind `"steward"`: a restart, a deploy, a stall), read from its own once-a-minute health sample. A plugin doesn't need to report those.
|
||||
|
||||
---
|
||||
|
||||
## Shipping a Plugin
|
||||
|
||||
@@ -758,7 +758,7 @@ async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dic
|
||||
from steward.mcp.tools._events import event
|
||||
rows = (await db.execute(select(DockerEvent).where(
|
||||
DockerEvent.host_id == host_id, DockerEvent.at >= since, DockerEvent.at <= until)
|
||||
.order_by(DockerEvent.at).limit(40))).scalars()
|
||||
.order_by(DockerEvent.at).limit(500))).scalars()
|
||||
return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
|
||||
subject=f"{host_name}/{e.container_name}") for e in rows]
|
||||
|
||||
|
||||
@@ -55,6 +55,16 @@ def last_slot(plan, now: datetime) -> datetime:
|
||||
return slot.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def fired_slot(plan) -> datetime | None:
|
||||
"""The last slot that fired, or None while none has. A new plan's
|
||||
last_slot_at holds the moment it was made, so its first slot is the
|
||||
next one rather than one already past; that moment is no slot (#5863)."""
|
||||
at = plan.last_slot_at
|
||||
if at is None:
|
||||
return None
|
||||
return at if last_slot(plan, at) == at else None
|
||||
|
||||
|
||||
def next_slot(plan, now: datetime) -> datetime:
|
||||
return last_slot(plan, now) + timedelta(days=7)
|
||||
|
||||
|
||||
@@ -136,7 +136,8 @@ async def list_maintenance_plans() -> dict:
|
||||
|
||||
A run updates the group's hosts one at a time; `latest_run` says how far
|
||||
it got (`hosts_done` of `hosts`), which hosts it passed over and why
|
||||
(`skipped`), and how it ended. get_maintenance_job shows one host's job.
|
||||
(`skipped`), and how it ended. `last_slot_at` is the last scheduled slot
|
||||
that fired, null until one has. get_maintenance_job shows one host's job.
|
||||
Plans are created and edited on the Maintenance page.
|
||||
"""
|
||||
from steward.models.ansible_inventory import AnsibleGroup
|
||||
@@ -164,7 +165,7 @@ async def list_maintenance_plans() -> dict:
|
||||
"weekday": cadence.WEEKDAYS[plan.weekday], "at_time": plan.at_time,
|
||||
"time_zone": plan.time_zone, "window_minutes": plan.window_minutes,
|
||||
"reboot_policy": plan.reboot_policy, "enabled": plan.enabled,
|
||||
"next_slot_at": iso(nxt), "last_slot_at": iso(plan.last_slot_at),
|
||||
"next_slot_at": iso(nxt), "last_slot_at": iso(cadence.fired_slot(plan)),
|
||||
"latest_run": None,
|
||||
}
|
||||
if run:
|
||||
@@ -303,7 +304,7 @@ async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dic
|
||||
jobs = (await db.execute(select(HostMaintenanceJob).where(
|
||||
HostMaintenanceJob.host_id == host_id, HostMaintenanceJob.created_at <= until,
|
||||
(HostMaintenanceJob.finished_at.is_(None)) | (HostMaintenanceJob.finished_at >= since))
|
||||
.order_by(HostMaintenanceJob.created_at).limit(20))).scalars()
|
||||
.order_by(HostMaintenanceJob.created_at).limit(200))).scalars()
|
||||
for job in jobs:
|
||||
out.append(event(job.created_at, "maintenance", f"Maintenance: {maintenance.summary(job)}",
|
||||
until=job.finished_at, ids={"job_id": job.id}))
|
||||
@@ -311,7 +312,7 @@ async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dic
|
||||
rows = (await db.execute(select(HostDowntime).where(
|
||||
HostDowntime.host_id == host_id, HostDowntime.started_at <= until,
|
||||
(HostDowntime.ended_at.is_(None)) | (HostDowntime.ended_at >= since))
|
||||
.order_by(HostDowntime.started_at).limit(20))).scalars()
|
||||
.order_by(HostDowntime.started_at).limit(200))).scalars()
|
||||
for row in rows:
|
||||
seen = downtime.describe(row, now)
|
||||
out.append(event(row.started_at, "downtime",
|
||||
|
||||
@@ -72,3 +72,10 @@ def register_mcp_tools(register) -> None:
|
||||
async def mcp_overview(db) -> list[dict]:
|
||||
from .mcp_tools import mcp_overview as findings
|
||||
return await findings(db)
|
||||
|
||||
|
||||
async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
|
||||
if source != "traefik":
|
||||
return []
|
||||
from .mcp_tools import service_events
|
||||
return await service_events(db, resource, since, until)
|
||||
|
||||
@@ -22,6 +22,7 @@ from steward.mcp.tools._shape import (
|
||||
from steward.mcp.tools.overview import finding
|
||||
from steward.models.users import UserRole
|
||||
|
||||
from . import meaning
|
||||
from . import routes as page
|
||||
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat
|
||||
from .scheduler import error_page_services, router_domains
|
||||
@@ -188,15 +189,20 @@ async def get_router(router: str, range: str = "24h") -> dict:
|
||||
return out
|
||||
|
||||
|
||||
async def list_certificates(include_old: bool = False) -> dict:
|
||||
async def list_certificates(include_old: bool = False, limit: int = 100) -> dict:
|
||||
"""The TLS certificates Traefik serves, soonest expiry first: names
|
||||
covered, expiry, days left, and the routers whose hosts each covers
|
||||
(null when Traefik's API couldn't say which hosts a router serves).
|
||||
(null when Traefik's API couldn't say which hosts a router serves; []
|
||||
when no current HTTP router serves its names, as for a certificate
|
||||
Traefik holds for a retired service, or one only a TCP router uses).
|
||||
|
||||
include_old: also list certificates Traefik stopped serving (renewed or
|
||||
retired), which Steward keeps on record.
|
||||
retired; a renewed one counts as old though Traefik still reports
|
||||
it), which Steward keeps on record.
|
||||
limit: certificates to return, default 100, max 500.
|
||||
`warn_days` / `crit_days` are the Settings → Thresholds cutoffs
|
||||
get_overview reports a served certificate at.
|
||||
get_overview reports a served certificate at. `unrouted` counts the
|
||||
served certificates with no router.
|
||||
"""
|
||||
thresholds, scrape = _cfg()
|
||||
now = now_utc()
|
||||
@@ -214,8 +220,11 @@ async def list_certificates(include_old: bool = False) -> dict:
|
||||
return sorted({n for n in names for h in page.router_hosts(n, domains) if covers(covered, h)})
|
||||
|
||||
tls = thresholds.get("tls_days", {})
|
||||
out = {"certificates": [_cert_view(c, now, routers_for(c)) for c in served],
|
||||
"warn_days": tls.get("warn", 14), "crit_days": tls.get("crit", 3)}
|
||||
views = [_cert_view(c, now, routers_for(c)) for c in served]
|
||||
out = bounded(views, clamp_limit(limit, 100, 500), key="certificates",
|
||||
more="Pass a larger `limit`.")
|
||||
out.update({"unrouted": sum(1 for v in views if v["routers"] == []),
|
||||
"warn_days": tls.get("warn", 14), "crit_days": tls.get("crit", 3)})
|
||||
if include_old:
|
||||
out["old"] = [_cert_view(c, now, None) | {"last_seen_at": iso(c.scraped_at)} for c in old[:50]]
|
||||
return out
|
||||
@@ -316,6 +325,29 @@ async def mcp_overview(db) -> list[dict]:
|
||||
return router_findings(current) + cert_findings(served, now, thresholds)
|
||||
|
||||
|
||||
async def service_events(db, router: str, since, until) -> list[dict]:
|
||||
"""What happened to the containers of the swarm service behind a router
|
||||
(starts with their image, deaths, OOM kills, health changes), on every
|
||||
host: a router anomaly has no host of its own to read events by (#5862).
|
||||
[] without the Docker plugin, or when no single service matches."""
|
||||
from steward.core.plugin_manager import loaded_plugin_submodule
|
||||
from steward.mcp.tools._events import event
|
||||
from steward.models.hosts import Host
|
||||
models = loaded_plugin_submodule("docker", "models")
|
||||
if models is None:
|
||||
return []
|
||||
service = meaning.swarm_service(router, await page.swarm_service_names(db))
|
||||
if service is None:
|
||||
return []
|
||||
e = models.DockerEvent
|
||||
rows = (await db.execute(
|
||||
select(e, Host.name).join(Host, Host.id == e.host_id)
|
||||
.where(e.container_name.startswith(f"{service}.", autoescape=True), e.at >= since, e.at <= until)
|
||||
.order_by(e.at).limit(500))).all()
|
||||
return [event(ev.at, "container", f"{ev.container_name} {ev.event}" + (f" ({ev.detail})" if ev.detail else ""),
|
||||
subject=f"{host}/{ev.container_name}") for ev, host in rows]
|
||||
|
||||
|
||||
def register_tools(register) -> None:
|
||||
for fn in (list_routers, get_router, list_certificates, get_traffic):
|
||||
register(fn, role=UserRole.viewer, read_only=True)
|
||||
|
||||
@@ -307,14 +307,22 @@ def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
|
||||
"""(served, old): a certificate is served while Traefik still reports it.
|
||||
Rows are upserted by serial and never deleted, so a renewed or retired
|
||||
certificate lingers; one missing from the last GONE_AFTER_SCRAPES
|
||||
scrapes is old (#5254)."""
|
||||
scrapes is old (#5254). So is one renewed: a Traefik that hasn't
|
||||
restarted since keeps reporting the certificate it replaced, so one
|
||||
whose every name a later-expiring reported certificate covers is old
|
||||
too (#5864); otherwise it reads as expiring while the renewal serves."""
|
||||
certs = list(certs)
|
||||
if not certs:
|
||||
return [], []
|
||||
newest = max(c.scraped_at for c in certs)
|
||||
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
|
||||
return ([c for c in certs if c.scraped_at >= cutoff],
|
||||
sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
|
||||
reported = [c for c in certs if c.scraped_at >= cutoff]
|
||||
names = {c.serial: cert_names(c.cn, c.sans) for c in reported}
|
||||
renewed = {c.serial for c in reported for n in reported
|
||||
if n.serial != c.serial and n.not_after > c.not_after and names[c.serial] <= names[n.serial]}
|
||||
return ([c for c in reported if c.serial not in renewed],
|
||||
sorted((c for c in certs if c.scraped_at < cutoff or c.serial in renewed),
|
||||
key=lambda c: c.scraped_at, reverse=True))
|
||||
|
||||
|
||||
def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
|
||||
|
||||
+10
-1
@@ -82,6 +82,9 @@ class StewardMCPServer(MCPServer):
|
||||
# plugin name -> its `mcp_events(db, host_id, host_name, since, until)`:
|
||||
# what it knows happened on a host, for get_anomaly.
|
||||
self.event_sources: dict[str, object] = {}
|
||||
# plugin name -> its `mcp_resource_events(db, source, resource, since,
|
||||
# until)`: what it knows happened to a resource with no host.
|
||||
self.resource_event_sources: dict[str, object] = {}
|
||||
|
||||
# ── registration ─────────────────────────────────────────────────────────
|
||||
def register_tool(self, fn, *, role: UserRole, read_only: bool, plugin: str | None = None) -> None:
|
||||
@@ -108,7 +111,9 @@ class StewardMCPServer(MCPServer):
|
||||
"""Add a plugin's tools via its optional `register_mcp_tools(register)`,
|
||||
its optional `mcp_overview(db)` as a get_overview findings source, and
|
||||
its optional `mcp_events(db, host_id, host_name, since, until)` as a
|
||||
source of what happened on a host (get_anomaly).
|
||||
source of what happened on a host (get_anomaly), and its optional
|
||||
`mcp_resource_events(db, source, resource, since, until)` for what
|
||||
happened to a resource with no host.
|
||||
|
||||
Idempotent per plugin (hot-reload drops its previous tools first) and
|
||||
tolerant: a raising hook is logged and leaves the plugin with no tools,
|
||||
@@ -123,6 +128,10 @@ class StewardMCPServer(MCPServer):
|
||||
events = getattr(module, "mcp_events", None)
|
||||
if callable(events):
|
||||
self.event_sources[name] = events
|
||||
self.resource_event_sources.pop(name, None)
|
||||
resource_events = getattr(module, "mcp_resource_events", None)
|
||||
if callable(resource_events):
|
||||
self.resource_event_sources[name] = resource_events
|
||||
hook = getattr(module, "register_mcp_tools", None)
|
||||
if hook is None:
|
||||
return
|
||||
|
||||
@@ -1,26 +1,32 @@
|
||||
"""What else was going on: events on a host, or one resource, in a window
|
||||
(#5837), so an anomaly comes with its circumstances.
|
||||
|
||||
Core gathers alert transitions, Ansible runs and other metric anomalies. A
|
||||
plugin adds its own through `mcp_events(db, host_id, host_name, since, until)`
|
||||
in its __init__.py (the host agent its maintenance jobs and downtime, Docker
|
||||
its container events); each returns a list of `event(...)`.
|
||||
Core gathers alert transitions, Ansible runs, other metric anomalies and
|
||||
Steward's own gaps in recording. A plugin adds its own through
|
||||
`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the
|
||||
host agent its maintenance jobs and downtime, Docker its container events),
|
||||
and for a resource with no host through
|
||||
`mcp_resource_events(db, source, resource, since, until)` (Traefik: the
|
||||
containers of the swarm service behind a router, #5862); each returns a list
|
||||
of `event(...)`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import and_, or_, select
|
||||
|
||||
from steward.alerts.service import watches_host
|
||||
from steward.core.alerts import host_of_resource
|
||||
from steward.core.formatting import human_duration
|
||||
from steward.mcp._context import mcp_app
|
||||
from steward.models.alerts import AlertEvent, AlertRule
|
||||
from steward.models.ansible import AnsibleRun
|
||||
from steward.models.ansible_inventory import AnsibleTarget
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.models.monitors import Monitor
|
||||
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
|
||||
|
||||
@@ -28,7 +34,11 @@ from ._shape import db_session, iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PER_SOURCE = 40 # events any one source contributes to one window
|
||||
PER_SOURCE = 40 # events any one source contributes to one window: the
|
||||
# nearest the occurrence's start, the rest counted (#5861)
|
||||
# Steward samples its own health once a minute (core/self_monitor.py); a
|
||||
# longer break between two samples is a stretch it wasn't recording.
|
||||
STEWARD_GAP_S = 90.0
|
||||
|
||||
|
||||
def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
|
||||
@@ -89,7 +99,7 @@ async def alert_events(db, place: Place, since: datetime, until: datetime) -> li
|
||||
note = f" ({ev.note})" if ev.note else ""
|
||||
out.append(event(ev.transitioned_at, "alert", f"{rule.name}: {ev.from_state} → {ev.to_state}{note}",
|
||||
subject=rule.resource_name, ids={"rule_id": rule.id}))
|
||||
return out[:PER_SOURCE]
|
||||
return out
|
||||
|
||||
|
||||
async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> list[dict]:
|
||||
@@ -110,7 +120,7 @@ async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> li
|
||||
out.append(event(r.started_at, "ansible_run",
|
||||
f"Ansible {r.playbook_path}{check}: {r.status.value}",
|
||||
until=r.finished_at, ids={"run_id": r.id}))
|
||||
return out[:PER_SOURCE]
|
||||
return out
|
||||
|
||||
|
||||
def _same_host(source: str, resource: str, place: Place) -> bool:
|
||||
@@ -139,12 +149,41 @@ async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
|
||||
until=span.ended_at, subject=a.resource_name, ids={"anomaly_id": a.id}))
|
||||
else:
|
||||
elsewhere += 1
|
||||
return here[:PER_SOURCE], elsewhere
|
||||
return here, elsewhere
|
||||
|
||||
|
||||
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[dict]:
|
||||
"""What enabled plugins know happened on the host (`mcp_events`), each
|
||||
in its own session so one that fails can't spoil the others' reads."""
|
||||
def gaps_in(times: list[datetime], since: datetime, until: datetime) -> list[dict]:
|
||||
"""Each break longer than STEWARD_GAP_S between consecutive samples that
|
||||
overlaps the window, as a "steward" event lasting the break."""
|
||||
out = []
|
||||
for before, after in zip(times, times[1:]):
|
||||
secs = (after - before).total_seconds()
|
||||
if secs > STEWARD_GAP_S and after >= since and before <= until:
|
||||
out.append(event(before, "steward",
|
||||
f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a "
|
||||
"stall): readings either side of a gap can be wrong", until=after))
|
||||
return out
|
||||
|
||||
|
||||
async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]:
|
||||
"""When Steward itself wasn't recording, read from its own once-a-minute
|
||||
health sample. Shown beside every anomaly, placed or not: a restart
|
||||
distorts whatever Steward measures, a Traefik counter as much as a host's
|
||||
silence (#5862). Raw samples only, so a window older than their
|
||||
retention shows none."""
|
||||
rows = (await db.execute(
|
||||
select(PluginMetric.recorded_at).where(
|
||||
PluginMetric.source_module == "steward", PluginMetric.resource_name == "process",
|
||||
PluginMetric.metric_name == "db_roundtrip_ms",
|
||||
PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until)
|
||||
.order_by(PluginMetric.recorded_at))).scalars()
|
||||
return gaps_in(list(rows), since, until)
|
||||
|
||||
|
||||
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]:
|
||||
"""What enabled plugins know happened on the host (`mcp_events`), one
|
||||
list per plugin, each read in its own session so one that fails can't
|
||||
spoil the others' reads."""
|
||||
server = getattr(mcp_app(), "mcp_server", None)
|
||||
if server is None or not place.host_id:
|
||||
return []
|
||||
@@ -154,17 +193,56 @@ async def plugin_events(place: Place, since: datetime, until: datetime) -> list[
|
||||
continue
|
||||
try:
|
||||
async with db_session() as db:
|
||||
out.extend((await source(db, place.host_id, place.host_name, since, until))[:PER_SOURCE])
|
||||
out.append(await source(db, place.host_id, place.host_name, since, until))
|
||||
except Exception:
|
||||
logger.exception("MCP events source of plugin %r failed", plugin)
|
||||
return out
|
||||
|
||||
|
||||
async def around(db, place: Place, anomaly_id: str, since: datetime,
|
||||
until: datetime) -> tuple[list[dict], int]:
|
||||
"""Everything above for one window, oldest first, and the count of
|
||||
anomalies elsewhere."""
|
||||
async def resource_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]:
|
||||
"""What enabled plugins know happened to the resource itself
|
||||
(`mcp_resource_events`), for one with no host to read events by: one
|
||||
list per plugin."""
|
||||
server = getattr(mcp_app(), "mcp_server", None)
|
||||
if server is None:
|
||||
return []
|
||||
out = []
|
||||
for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()):
|
||||
if not server.plugin_enabled(plugin):
|
||||
continue
|
||||
try:
|
||||
async with db_session() as db:
|
||||
out.append(await source(db, place.source, place.resource, since, until))
|
||||
except Exception:
|
||||
logger.exception("MCP resource events source of plugin %r failed", plugin)
|
||||
return out
|
||||
|
||||
|
||||
def _at(e: dict) -> datetime | None:
|
||||
return datetime.fromisoformat(e["at"].replace("Z", "+00:00")) if e.get("at") else None
|
||||
|
||||
|
||||
def nearest(events: list[dict], focus: datetime, keep: int = PER_SOURCE) -> tuple[list[dict], int]:
|
||||
"""The `keep` events nearest `focus` (an occurrence's start), and how
|
||||
many were left out: a busy host's window is cut where it matters least."""
|
||||
if len(events) <= keep:
|
||||
return events, 0
|
||||
by_distance = sorted(events, key=lambda e: abs((_at(e) - focus).total_seconds()) if _at(e) else float("inf"))
|
||||
return by_distance[:keep], len(events) - keep
|
||||
|
||||
|
||||
async def around(db, place: Place, anomaly_id: str, since: datetime, until: datetime,
|
||||
focus: datetime) -> tuple[list[dict], int, int]:
|
||||
"""Everything above for one window, oldest first; the count of anomalies
|
||||
elsewhere; and how many events were left out, each source keeping its
|
||||
PER_SOURCE nearest `focus`."""
|
||||
anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
|
||||
found = (await alert_events(db, place, since, until) + await ansible_runs(db, place, since, until)
|
||||
+ anomalies + await plugin_events(place, since, until))
|
||||
return sorted(found, key=lambda e: e["at"] or ""), elsewhere
|
||||
groups = [await alert_events(db, place, since, until), await ansible_runs(db, place, since, until),
|
||||
anomalies, *await plugin_events(place, since, until),
|
||||
*await resource_events(place, since, until), await steward_gaps(db, since, until)]
|
||||
found, left_out = [], 0
|
||||
for group in groups:
|
||||
kept, cut = nearest(group, focus)
|
||||
found += kept
|
||||
left_out += cut
|
||||
return sorted(found, key=lambda e: e["at"] or ""), elsewhere, left_out
|
||||
|
||||
@@ -158,9 +158,14 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
|
||||
- `occurrences[]`: `started_at`, `ended_at`, `held_s`, `peak_level`,
|
||||
and `usual` for that hour as Steward has learned it now (`usual_hold`,
|
||||
`usual_peak`, `flag_above`), plus `around`: alert transitions, Ansible
|
||||
runs, other anomalies on the host, and what plugins saw (container
|
||||
runs, other anomalies on the host, what plugins saw (container
|
||||
starts, deaths and OOM kills; maintenance jobs; reboots and agent
|
||||
stops), from 30 minutes before to 30 minutes after; and
|
||||
stops; for a Traefik router, its swarm service's containers on any
|
||||
host), and kind "steward": a stretch Steward itself wasn't recording
|
||||
(a restart or deploy), around which any reading can be wrong. From 30
|
||||
minutes before to 30 minutes after. Each kind keeps the 40 events
|
||||
nearest the occurrence's start; `around_left_out` counts the rest
|
||||
(0 when nothing was cut). And
|
||||
`anomalies_elsewhere`, how many anomalies on other hosts overlapped it.
|
||||
Many at once points at something shared (network, storage, Steward).
|
||||
- `series`: the newest shown occurrence with time either side (as long
|
||||
@@ -191,8 +196,8 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
|
||||
shown = []
|
||||
for span in spans[:limit]:
|
||||
usual = _usual_at(kind, slots, span.started_at)
|
||||
near, elsewhere = await events.around(db, place, a.id, span.started_at - AROUND,
|
||||
(span.ended_at or now) + AROUND)
|
||||
near, elsewhere, left_out = await events.around(db, place, a.id, span.started_at - AROUND,
|
||||
(span.ended_at or now) + AROUND, span.started_at)
|
||||
shown.append({
|
||||
"started_at": iso(span.started_at), "ended_at": iso(span.ended_at),
|
||||
"ongoing": span.ended_at is None,
|
||||
@@ -203,6 +208,7 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
|
||||
"flag_above": rnd(flag_above(usual.usual_hold, floor), 3) if floor is not None else None}
|
||||
if usual else None),
|
||||
"around": near,
|
||||
"around_left_out": left_out,
|
||||
"anomalies_elsewhere": elsewhere,
|
||||
})
|
||||
series = None
|
||||
|
||||
@@ -103,3 +103,37 @@ def test_a_plugin_adds_events_through_its_hook(app):
|
||||
assert "events_test" not in mcp.event_sources
|
||||
finally:
|
||||
mcp.event_sources.pop("events_test", None)
|
||||
|
||||
|
||||
def test_a_plugin_adds_a_hostless_resources_events_through_its_hook(app):
|
||||
async def events(db, source, resource, since, until):
|
||||
return []
|
||||
|
||||
mcp = app.mcp_server
|
||||
mcp.register_plugin("events_test", NS(mcp_resource_events=events))
|
||||
try:
|
||||
assert mcp.resource_event_sources["events_test"] is events
|
||||
mcp.register_plugin("events_test", NS())
|
||||
assert "events_test" not in mcp.resource_event_sources
|
||||
finally:
|
||||
mcp.resource_event_sources.pop("events_test", None)
|
||||
|
||||
|
||||
def test_a_busy_window_keeps_the_events_nearest_the_occurrence_and_counts_the_rest():
|
||||
"""#5861: 50 container events over an hour; the occurrence starts at :45."""
|
||||
evs = [_events.event(NOW + timedelta(minutes=m), "container", f"c{m} start") for m in range(50)]
|
||||
kept, cut = _events.nearest(evs, NOW + timedelta(minutes=45))
|
||||
assert len(kept) == 40 and cut == 10
|
||||
assert {e["what"] for e in kept} >= {"c45 start", "c49 start", "c20 start"}
|
||||
assert "c0 start" not in {e["what"] for e in kept}
|
||||
assert _events.nearest(evs[:5], NOW) == (evs[:5], 0)
|
||||
|
||||
|
||||
def test_steward_not_recording_is_an_event_beside_every_anomaly():
|
||||
"""A Steward restart between two of its once-a-minute samples (#5862)."""
|
||||
ticks = [NOW + timedelta(minutes=m) for m in (0, 1, 2)] + [NOW + timedelta(minutes=4, seconds=10)]
|
||||
[gap] = _events.gaps_in(ticks, NOW, NOW + timedelta(hours=1))
|
||||
assert gap["kind"] == "steward" and gap["at"] == "2026-10-15T12:02:00Z"
|
||||
assert gap["until"] == "2026-10-15T12:04:10Z" and "wasn't recording" in gap["what"]
|
||||
assert _events.gaps_in(ticks, NOW + timedelta(minutes=5), NOW + timedelta(hours=1)) == []
|
||||
assert _events.gaps_in([NOW + timedelta(minutes=m) for m in range(5)], NOW, NOW + timedelta(hours=1)) == []
|
||||
|
||||
@@ -21,6 +21,16 @@ def _plan(**kw):
|
||||
return NS(**base)
|
||||
|
||||
|
||||
def test_a_plan_that_has_not_fired_has_no_last_slot():
|
||||
"""#5863: a new plan's last_slot_at is the moment it was made, a Wednesday
|
||||
evening for a Sunday plan; only a slot that fired is reported."""
|
||||
made = _plan(time_zone="America/New_York", last_slot_at=datetime(2026, 10, 14, 23, 54, 58, tzinfo=timezone.utc))
|
||||
assert c.fired_slot(made) is None
|
||||
fired = datetime(2026, 10, 18, 7, 0, tzinfo=timezone.utc) # Sunday 03:00 New York
|
||||
assert c.fired_slot(_plan(time_zone="America/New_York", last_slot_at=fired)) == fired
|
||||
assert c.fired_slot(_plan()) is None
|
||||
|
||||
|
||||
def test_the_last_slot_is_the_latest_weekday_and_time_in_the_plans_zone():
|
||||
assert c.last_slot(_plan(), NOW) == datetime(2026, 10, 11, 3, 0, tzinfo=timezone.utc)
|
||||
# 03:00 in New York is 07:00 UTC in October (EDT)
|
||||
|
||||
@@ -175,6 +175,19 @@ def test_a_certificate_traefik_stopped_reporting_is_old_not_current():
|
||||
assert current_certs([], 60) == ([], [])
|
||||
|
||||
|
||||
def test_a_certificate_a_later_one_renewed_is_old_though_traefik_still_reports_it():
|
||||
"""codex.fabledsword.com on 2026-10-15: the replaced certificate (21 days
|
||||
left) still reported beside its renewal (81 days), #5864."""
|
||||
from plugins.traefik.routes import current_certs
|
||||
old_one, renewal = _cert("1", "codex.example", 21), _cert("2", "codex.example", 81)
|
||||
wider = _cert("3", "a.example", 90, sans="a.example,b.example")
|
||||
narrower = _cert("4", "b.example", 30)
|
||||
unrelated = _cert("5", "c.example", 10)
|
||||
served, old = current_certs([old_one, renewal, wider, narrower, unrelated], 60)
|
||||
assert {c.serial for c in served} == {"2", "3", "5"}
|
||||
assert {c.serial for c in old} == {"1", "4"}
|
||||
|
||||
|
||||
def test_each_router_gets_the_soonest_certificate_for_its_hosts():
|
||||
from plugins.traefik.routes import certs_by_router
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
Reference in New Issue
Block a user