bvandeusenandClaude Opus 5.5
5a0e150789
#5752 "Container CPU and memory history lives in the one metric store; docker_metrics and docker_metrics_hourly are retired". Container cpu_pct and mem_pct already went to plugin_metrics for alerting; now mem_usage_bytes does too, and every snapshot in an ingest (backlog included) is recorded at its own time. The container history chart, Busiest Containers, a service's history and the MCP container tools read plugin_metrics under resource "<host>/<container>", rolled up and pruned by the shared metric retention. docker_012_drop_metrics drops both tables. Their history is not copied (lesson #5221 "A data migration's cost is set by the live table"); the store already holds 7 days of container CPU and memory %. Docker retention now prunes only lifecycle events, and Settings -> Thresholds has one "Metric retention" card for every source. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
153 lines
7.9 KiB
Python
153 lines
7.9 KiB
Python
"""Integration: a swarm service's own page (#5302) and the Docker overview's
|
|
service table (#5299, #5300) against real Postgres. The page reads the logs
|
|
and samples of every task the service ran, the live one and one Docker has
|
|
already removed (known only by its events), and a failure's "Logs →" picks
|
|
that task. Requires STEWARD_DATABASE_URL."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import importlib
|
|
import json
|
|
import os
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def _docker(sub):
|
|
"""The app's loaded docker plugin module (see test_docker_recent_usage)."""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
loaded = get_loaded_plugin_modules().get("docker")
|
|
base = loaded.__name__ if loaded is not None else "plugins.docker"
|
|
return importlib.import_module(f"{base}.{sub}")
|
|
|
|
|
|
def test_a_service_page_reads_every_task(app):
|
|
from steward.models.hosts import Host
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
m = _docker("models")
|
|
|
|
tag = uuid.uuid4().hex[:8]
|
|
name = f"it{tag}_web"
|
|
live_task, dead_task = "a" * 17 + tag, "b" * 17 + tag # 25 characters, as Swarm's
|
|
live, dead = f"{name}.1.{live_task}", f"{name}.1.{dead_task}"
|
|
hid, uid, node = str(uuid.uuid4()), str(uuid.uuid4()), f"n{tag}"
|
|
now = datetime.now(timezone.utc)
|
|
|
|
def iso(dt):
|
|
return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
|
|
tasks = [{"task_id": live_task, "slot": 1, "node_id": node, "desired_state": "running",
|
|
"state": "running", "message": "started", "error": "", "exit_code": None,
|
|
"at": iso(now - timedelta(minutes=9))},
|
|
{"task_id": dead_task, "slot": 1, "node_id": node, "desired_state": "shutdown",
|
|
"state": "failed", "message": "started", "error": "task: non-zero exit (1)",
|
|
"exit_code": 1, "at": iso(now - timedelta(minutes=10))}]
|
|
|
|
def sample(cname, mins, cpu, mem):
|
|
when = now - timedelta(minutes=mins)
|
|
return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{cname}",
|
|
metric_name=metric, value=value, recorded_at=when)
|
|
for metric, value in (("cpu_pct", cpu), ("mem_usage_bytes", mem))]
|
|
|
|
async def go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add(User(id=uid, username=f"sp-{tag}", email=f"sp-{tag}@test",
|
|
password_hash="x", role=UserRole.viewer, is_active=True))
|
|
s.add(Host(id=hid, name=f"dock-{tag}", address="10.9.7.1"))
|
|
async with s.begin():
|
|
s.add_all([
|
|
m.DockerSwarmNode(host_id=hid, node_id=node, hostname=f"node-{tag}", role="manager",
|
|
availability="active", status="ready", leader=True),
|
|
m.DockerSwarmService(host_id=hid, service_name=name, mode="replicated", desired=1,
|
|
running=1, image="nginx:1.27",
|
|
placement_json=json.dumps([{"node_id": node, "running": 1}]),
|
|
tasks_json=json.dumps(tasks), updated_at=now),
|
|
m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running",
|
|
cpu_pct=50.0, mem_usage_bytes=100 << 20, service_name=name,
|
|
task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9),
|
|
scraped_at=now),
|
|
# Docker has removed the failed task's container: only its events are left.
|
|
m.DockerEvent(host_id=hid, container_name=dead, event="start", at=now - timedelta(minutes=11)),
|
|
m.DockerEvent(host_id=hid, container_name=dead, event="die", detail="exit 1",
|
|
at=now - timedelta(minutes=10)),
|
|
m.DockerLog(host_id=hid, container_name=live, stream="stdout", line=f"serving {tag}",
|
|
ts=now - timedelta(minutes=5)),
|
|
m.DockerLog(host_id=hid, container_name=dead, stream="stderr", line=f"panic {tag}",
|
|
ts=now - timedelta(minutes=10)),
|
|
])
|
|
s.add_all(sample(dead, 40, 200.0, 300 << 20) + sample(live, 30, 40.0, 90 << 20)
|
|
+ sample(live, 20, 60.0, 100 << 20) + sample(live, 1, 50.0, 100 << 20))
|
|
|
|
client = app.test_client()
|
|
async with client.session_transaction() as sess:
|
|
sess.update(user_id=uid, user_role="viewer", username=f"sp-{tag}", login_at=time.time())
|
|
|
|
async def get(url):
|
|
resp = await client.get(url)
|
|
return resp.status_code, await resp.get_data(as_text=True)
|
|
|
|
out = {"page": await get(f"/plugins/docker/service/{name}"),
|
|
"focus": await get(f"/plugins/docker/service/{name}?task={dead_task}"),
|
|
"missing": await get(f"/plugins/docker/service/it{tag}_nothing"),
|
|
"logs": await get(f"/plugins/docker/service/{name}/logs/lines"),
|
|
"one": await get(f"/plugins/docker/service/{name}/logs/lines?task={dead_task}"),
|
|
"history": await get(f"/plugins/docker/service/{name}/history?range=1h"),
|
|
"rows": await get(f"/plugins/docker/rows?q=it{tag}"),
|
|
"other": await get(f"/plugins/docker/rows?q=it{tag}&stack=someotherstack")}
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.delete(await s.get(Host, hid))
|
|
await s.delete(await s.get(User, uid))
|
|
return out
|
|
|
|
got = asyncio.run(go())
|
|
|
|
status, page = got["page"]
|
|
assert status == 200
|
|
assert "nginx:1.27" in page and "1 of 1 running" in page
|
|
assert "task: non-zero exit (1)" in page # Swarm's task record
|
|
# One table (#5310): the node reads as the Steward host on it, on the running
|
|
# task and the failed one alike; the running task links its container.
|
|
assert page.count(f'<a href="/hosts/{hid}">dock-{tag}</a>') == 2 and f"node-{tag}" not in page
|
|
assert f'href="/plugins/docker/container/{hid}/{live}"' in page
|
|
assert "in trouble" not in page and "Last failed task" not in page # it is healthy again
|
|
assert f'<option value="{live_task}">1 · {live_task[:10]}</option>' in page
|
|
assert f'<option value="{dead_task}">1 · {dead_task[:10]}</option>' in page
|
|
assert f'href="/plugins/docker/?stack=it{tag}"' in page
|
|
assert f'<option value="{dead_task}" selected>' in got["focus"][1]
|
|
assert got["missing"][0] == 404 and "Service not found" in got["missing"][1]
|
|
|
|
logs = got["logs"][1]
|
|
assert f"serving {tag}" in logs and f"panic {tag}" in logs # the removed task's lines too
|
|
assert logs.index(f"serving {tag}") < logs.index(f"panic {tag}") # newest first
|
|
assert f"1 · {dead_task[:10]}" in logs
|
|
one = got["one"][1]
|
|
assert f"panic {tag}" in one and f"serving {tag}" not in one
|
|
|
|
history = got["history"][1]
|
|
assert "<polyline" in history and "peak 2.0 cores" in history and "now · peak" in history
|
|
|
|
rows = got["rows"][1]
|
|
assert f'href="/plugins/docker/service/{name}"' in rows
|
|
# A stack chip keeps the search it was clicked under.
|
|
assert f'<a class="chip" href="/plugins/docker/?q=it{tag}&stack=it{tag}"' in rows
|
|
assert f'/plugins/docker/service/{name}?task={dead_task}#logs' in rows # the failure's logs
|
|
# Another stack's filter leaves it out of the table; the failures list is never filtered.
|
|
assert f'<td class="cell-name"><a href="/plugins/docker/service/{name}"' in rows
|
|
assert f'<td class="cell-name"><a href="/plugins/docker/service/{name}"' not in got["other"][1]
|