#6718 "A swarm container's metrics, normal and anomalies are keyed by its task name, so they start over whenever the task moves". The operator: "the metrics should be keys to the service name not the task name... as that's what's consistent between updates." CPU, memory, restart count and health now go under "swarm/<service>", so a normal, its anomalies and an alert rule carry across a redeploy or a move between nodes. Network rates stay "<host>/<service>", the task id gone, as they answer who on a host is sending. Tasks of a multi-task service share the key, so its level reads as an average across them (of 51 live services, only authentik-worker and portainer_agent run more than one task). History under task names moves in a background task, 5000 raw samples every 15 s, not a migration: the raw table holds about 2M such rows and the app would wait on it at boot. Hourly rows combine where two tasks share an hour, and rows still to review merge to one per metric. A service's anomalies no longer reach a host's page, as it has no one host; get_anomaly reads what happened around one from the service's own events. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
123 lines
5.3 KiB
Python
123 lines
5.3 KiB
Python
# plugins/docker/__init__.py
|
|
from __future__ import annotations
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
_app: "Quart | None" = None
|
|
|
|
|
|
def setup(app: "Quart") -> None:
|
|
global _app
|
|
_app = app
|
|
from .models import DockerContainer # noqa: F401 — register with Base
|
|
|
|
# Publish the per-host persist hook so the host agent can hand off the
|
|
# `docker` array from each sample without importing our models (opportunistic
|
|
# synergy — host_agent gates on has_capability and no-ops if we're disabled).
|
|
# required_role=viewer: this is a trusted server-side data-plane write driven
|
|
# by the authenticated agent ingest, not a user-facing privileged action.
|
|
from steward.core.capabilities import register_capability
|
|
from steward.models.users import UserRole
|
|
from .ingest import persist_host_docker
|
|
from .retention import prune_docker_logs, run_docker_retention
|
|
register_capability(
|
|
"docker.persist_host_samples", persist_host_docker,
|
|
label="Persist host Docker samples",
|
|
description="Store per-host container state + metrics pushed by the host agent.",
|
|
required_role=UserRole.viewer,
|
|
)
|
|
# Prune lifecycle events, driven by the core cleanup task without it
|
|
# importing our models. Same trusted server-side data-plane role as above.
|
|
# Container CPU and memory live in the one metric store and its retention.
|
|
register_capability(
|
|
"docker.run_retention", run_docker_retention,
|
|
label="Run Docker retention",
|
|
description="Prune Docker lifecycle events past their window.",
|
|
required_role=UserRole.viewer,
|
|
)
|
|
register_capability(
|
|
"docker.prune_logs", prune_docker_logs,
|
|
label="Prune Docker logs",
|
|
description="Keep each container's logs within the age window and byte cap.",
|
|
required_role=UserRole.viewer,
|
|
)
|
|
# Each container's CPU (100% is one core) and memory share learn their
|
|
# normal (#5742), each with its floor from Settings → Thresholds. Memory
|
|
# is its processes' own, file cache left out (#6030).
|
|
from steward.core.norms import floor_setting, learn_normal
|
|
learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct"))
|
|
learn_normal("docker", "mem_used_pct", floor_setting(app, "container_mem_pct"))
|
|
# A host's agent carries its containers' figures: they go silent with it
|
|
# (#5721). A swarm service's aren't tied to one host.
|
|
from steward.core.alerts import agent_fed
|
|
from .service import SERVICE_RESOURCE
|
|
agent_fed("docker", lambda resource: None if resource.startswith(SERVICE_RESOURCE)
|
|
else resource.split("/", 1)[0])
|
|
# An alert on a swarm figure names the services or nodes it counts (#6400).
|
|
from steward.core.alerts import names_behind
|
|
from .swarm_health import NAMERS
|
|
for metric, names in NAMERS.items():
|
|
names_behind("docker", metric, names)
|
|
# Deploys and crash-loops go on the timeline.
|
|
from .timeline import register as register_timeline
|
|
register_timeline()
|
|
|
|
|
|
def get_scheduled_tasks() -> list:
|
|
# Collection is agent-driven (pushed to the host_agent ingest). The one
|
|
# task here moves task-named history to its service.
|
|
from .rekey import make_rekey_task
|
|
return [make_rekey_task(_app)]
|
|
|
|
|
|
def get_blueprint():
|
|
from .routes import docker_bp
|
|
return docker_bp
|
|
|
|
|
|
def register_mcp_tools(register) -> None:
|
|
from .mcp_tools import register_tools
|
|
register_tools(register)
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
from .mcp_tools import mcp_overview as findings
|
|
return await findings(db)
|
|
|
|
|
|
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
|
|
from .mcp_tools import mcp_events as events
|
|
return await events(db, host_id, host_name, since, until)
|
|
|
|
|
|
async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
|
|
from .service import SERVICE_RESOURCE
|
|
if source != "docker" or not resource.startswith(SERVICE_RESOURCE):
|
|
return []
|
|
from .mcp_tools import service_events
|
|
return await service_events(db, resource[len(SERVICE_RESOURCE):], since, until)
|
|
|
|
|
|
def get_nav() -> list[dict]:
|
|
# Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
|
|
return [{"label": "Docker", "href": "/plugins/docker/"}]
|
|
|
|
|
|
def get_alert_metrics() -> dict[str, list[str]]:
|
|
"""What ingest.py records per container (resource "swarm/<service>" for a
|
|
swarm task, else "<host>/<container>"): restart_count is cumulative (alert
|
|
on crash-looping); is_healthy is 1.0 / 0.0 from the container's HEALTHCHECK,
|
|
only for containers that define one. net_rx_bps / net_tx_bps are the bytes
|
|
a second it receives and sends, under "<host>/<service or container>"
|
|
(#6115): "a container flooding the uplink" is net_tx_bps over a level.
|
|
Per host (resource "<host>"), what Docker keeps on its disk (#5304).
|
|
Swarm-wide (resource "swarm"), from any manager's report: services short
|
|
of their replicas, nodes down and managers unreachable, drained nodes
|
|
aside (#5788, swarm_health.py)."""
|
|
from .disk_view import DISK_METRICS
|
|
from .swarm_health import SWARM_METRICS
|
|
return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", "net_rx_bps", "net_tx_bps",
|
|
*DISK_METRICS, *SWARM_METRICS]}
|