Files
FabledSteward/plugins/docker/__init__.py
T
bvandeusenandClaude Opus 5.5 db39fc2604 feat(docker): a swarm service's metrics stay under one key across its tasks (#6718)
#6718 "A swarm container's metrics, normal and anomalies are keyed by its task
name, so they start over whenever the task moves". The operator: "the metrics
should be keys to the service name not the task name... as that's what's
consistent between updates."

CPU, memory, restart count and health now go under "swarm/<service>", so a
normal, its anomalies and an alert rule carry across a redeploy or a move
between nodes. Network rates stay "<host>/<service>", the task id gone, as they
answer who on a host is sending. Tasks of a multi-task service share the key,
so its level reads as an average across them (of 51 live services, only
authentik-worker and portainer_agent run more than one task).

History under task names moves in a background task, 5000 raw samples every
15 s, not a migration: the raw table holds about 2M such rows and the app
would wait on it at boot. Hourly rows combine where two tasks share an hour,
and rows still to review merge to one per metric.

A service's anomalies no longer reach a host's page, as it has no one host;
get_anomaly reads what happened around one from the service's own events.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 19:45:16 -04:00

123 lines
5.3 KiB
Python

# plugins/docker/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
_app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import DockerContainer # noqa: F401 — register with Base
# Publish the per-host persist hook so the host agent can hand off the
# `docker` array from each sample without importing our models (opportunistic
# synergy — host_agent gates on has_capability and no-ops if we're disabled).
# required_role=viewer: this is a trusted server-side data-plane write driven
# by the authenticated agent ingest, not a user-facing privileged action.
from steward.core.capabilities import register_capability
from steward.models.users import UserRole
from .ingest import persist_host_docker
from .retention import prune_docker_logs, run_docker_retention
register_capability(
"docker.persist_host_samples", persist_host_docker,
label="Persist host Docker samples",
description="Store per-host container state + metrics pushed by the host agent.",
required_role=UserRole.viewer,
)
# Prune lifecycle events, driven by the core cleanup task without it
# importing our models. Same trusted server-side data-plane role as above.
# Container CPU and memory live in the one metric store and its retention.
register_capability(
"docker.run_retention", run_docker_retention,
label="Run Docker retention",
description="Prune Docker lifecycle events past their window.",
required_role=UserRole.viewer,
)
register_capability(
"docker.prune_logs", prune_docker_logs,
label="Prune Docker logs",
description="Keep each container's logs within the age window and byte cap.",
required_role=UserRole.viewer,
)
# Each container's CPU (100% is one core) and memory share learn their
# normal (#5742), each with its floor from Settings → Thresholds. Memory
# is its processes' own, file cache left out (#6030).
from steward.core.norms import floor_setting, learn_normal
learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct"))
learn_normal("docker", "mem_used_pct", floor_setting(app, "container_mem_pct"))
# A host's agent carries its containers' figures: they go silent with it
# (#5721). A swarm service's aren't tied to one host.
from steward.core.alerts import agent_fed
from .service import SERVICE_RESOURCE
agent_fed("docker", lambda resource: None if resource.startswith(SERVICE_RESOURCE)
else resource.split("/", 1)[0])
# An alert on a swarm figure names the services or nodes it counts (#6400).
from steward.core.alerts import names_behind
from .swarm_health import NAMERS
for metric, names in NAMERS.items():
names_behind("docker", metric, names)
# Deploys and crash-loops go on the timeline.
from .timeline import register as register_timeline
register_timeline()
def get_scheduled_tasks() -> list:
# Collection is agent-driven (pushed to the host_agent ingest). The one
# task here moves task-named history to its service.
from .rekey import make_rekey_task
return [make_rekey_task(_app)]
def get_blueprint():
from .routes import docker_bp
return docker_bp
def register_mcp_tools(register) -> None:
from .mcp_tools import register_tools
register_tools(register)
async def mcp_overview(db) -> list[dict]:
from .mcp_tools import mcp_overview as findings
return await findings(db)
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
from .mcp_tools import mcp_events as events
return await events(db, host_id, host_name, since, until)
async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
from .service import SERVICE_RESOURCE
if source != "docker" or not resource.startswith(SERVICE_RESOURCE):
return []
from .mcp_tools import service_events
return await service_events(db, resource[len(SERVICE_RESOURCE):], since, until)
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
return [{"label": "Docker", "href": "/plugins/docker/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""What ingest.py records per container (resource "swarm/<service>" for a
swarm task, else "<host>/<container>"): restart_count is cumulative (alert
on crash-looping); is_healthy is 1.0 / 0.0 from the container's HEALTHCHECK,
only for containers that define one. net_rx_bps / net_tx_bps are the bytes
a second it receives and sends, under "<host>/<service or container>"
(#6115): "a container flooding the uplink" is net_tx_bps over a level.
Per host (resource "<host>"), what Docker keeps on its disk (#5304).
Swarm-wide (resource "swarm"), from any manager's report: services short
of their replicas, nodes down and managers unreachable, drained nodes
aside (#5788, swarm_health.py)."""
from .disk_view import DISK_METRICS
from .swarm_health import SWARM_METRICS
return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", "net_rx_bps", "net_tx_bps",
*DISK_METRICS, *SWARM_METRICS]}