bvandeusenandClaude Opus 5.5
56063f27a4
The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
153 lines
7.9 KiB
Python
153 lines
7.9 KiB
Python
"""Integration: a swarm service's own page (#5302) and the Docker overview's
|
|
service table (#5299, #5300) against real Postgres. The page reads the logs
|
|
and samples of every task the service ran, the live one and one Docker has
|
|
already removed (known only by its events), and a failure's "Logs →" picks
|
|
that task. Requires STEWARD_DATABASE_URL."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import importlib
|
|
import json
|
|
import os
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def _docker(sub):
|
|
"""The app's loaded docker plugin module (see test_docker_recent_usage)."""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
loaded = get_loaded_plugin_modules().get("docker")
|
|
base = loaded.__name__ if loaded is not None else "plugins.docker"
|
|
return importlib.import_module(f"{base}.{sub}")
|
|
|
|
|
|
def test_a_service_page_reads_every_task(app):
|
|
from steward.models.hosts import Host
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
m = _docker("models")
|
|
|
|
tag = uuid.uuid4().hex[:8]
|
|
name = f"it{tag}_web"
|
|
live_task, dead_task = "a" * 17 + tag, "b" * 17 + tag # 25 characters, as Swarm's
|
|
live, dead = f"{name}.1.{live_task}", f"{name}.1.{dead_task}"
|
|
hid, uid, node = str(uuid.uuid4()), str(uuid.uuid4()), f"n{tag}"
|
|
now = datetime.now(timezone.utc)
|
|
|
|
def iso(dt):
|
|
return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
|
|
tasks = [{"task_id": live_task, "slot": 1, "node_id": node, "desired_state": "running",
|
|
"state": "running", "message": "started", "error": "", "exit_code": None,
|
|
"at": iso(now - timedelta(minutes=9))},
|
|
{"task_id": dead_task, "slot": 1, "node_id": node, "desired_state": "shutdown",
|
|
"state": "failed", "message": "started", "error": "task: non-zero exit (1)",
|
|
"exit_code": 1, "at": iso(now - timedelta(minutes=10))}]
|
|
|
|
def sample(cname, mins, cpu, mem):
|
|
when = now - timedelta(minutes=mins)
|
|
return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{cname}",
|
|
metric_name=metric, value=value, recorded_at=when)
|
|
for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))]
|
|
|
|
async def go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add(User(id=uid, username=f"sp-{tag}", email=f"sp-{tag}@test",
|
|
password_hash="x", role=UserRole.viewer, is_active=True))
|
|
s.add(Host(id=hid, name=f"dock-{tag}", address="10.9.7.1"))
|
|
async with s.begin():
|
|
s.add_all([
|
|
m.DockerSwarmNode(host_id=hid, node_id=node, hostname=f"node-{tag}", role="manager",
|
|
availability="active", status="ready", leader=True),
|
|
m.DockerSwarmService(host_id=hid, service_name=name, mode="replicated", desired=1,
|
|
running=1, image="nginx:1.27",
|
|
placement_json=json.dumps([{"node_id": node, "running": 1}]),
|
|
tasks_json=json.dumps(tasks), updated_at=now),
|
|
m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running",
|
|
cpu_pct=50.0, mem_used_bytes=100 << 20, service_name=name,
|
|
task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9),
|
|
scraped_at=now),
|
|
# Docker has removed the failed task's container: only its events are left.
|
|
m.DockerEvent(host_id=hid, container_name=dead, event="start", at=now - timedelta(minutes=11)),
|
|
m.DockerEvent(host_id=hid, container_name=dead, event="die", detail="exit 1",
|
|
at=now - timedelta(minutes=10)),
|
|
m.DockerLog(host_id=hid, container_name=live, stream="stdout", line=f"serving {tag}",
|
|
ts=now - timedelta(minutes=5)),
|
|
m.DockerLog(host_id=hid, container_name=dead, stream="stderr", line=f"panic {tag}",
|
|
ts=now - timedelta(minutes=10)),
|
|
])
|
|
s.add_all(sample(dead, 40, 200.0, 300 << 20) + sample(live, 30, 40.0, 90 << 20)
|
|
+ sample(live, 20, 60.0, 100 << 20) + sample(live, 1, 50.0, 100 << 20))
|
|
|
|
client = app.test_client()
|
|
async with client.session_transaction() as sess:
|
|
sess.update(user_id=uid, user_role="viewer", username=f"sp-{tag}", login_at=time.time())
|
|
|
|
async def get(url):
|
|
resp = await client.get(url)
|
|
return resp.status_code, await resp.get_data(as_text=True)
|
|
|
|
out = {"page": await get(f"/plugins/docker/service/{name}"),
|
|
"focus": await get(f"/plugins/docker/service/{name}?task={dead_task}"),
|
|
"missing": await get(f"/plugins/docker/service/it{tag}_nothing"),
|
|
"logs": await get(f"/plugins/docker/service/{name}/logs/lines"),
|
|
"one": await get(f"/plugins/docker/service/{name}/logs/lines?task={dead_task}"),
|
|
"history": await get(f"/plugins/docker/service/{name}/history?range=1h"),
|
|
"rows": await get(f"/plugins/docker/rows?q=it{tag}"),
|
|
"other": await get(f"/plugins/docker/rows?q=it{tag}&stack=someotherstack")}
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.delete(await s.get(Host, hid))
|
|
await s.delete(await s.get(User, uid))
|
|
return out
|
|
|
|
got = asyncio.run(go())
|
|
|
|
status, page = got["page"]
|
|
assert status == 200
|
|
assert "nginx:1.27" in page and "1 of 1 running" in page
|
|
assert "task: non-zero exit (1)" in page # Swarm's task record
|
|
# One table (#5310): the node reads as the Steward host on it, on the running
|
|
# task and the failed one alike; the running task links its container.
|
|
assert page.count(f'<a href="/hosts/{hid}">dock-{tag}</a>') == 2 and f"node-{tag}" not in page
|
|
assert f'href="/plugins/docker/container/{hid}/{live}"' in page
|
|
assert "in trouble" not in page and "Last failed task" not in page # it is healthy again
|
|
assert f'<option value="{live_task}">1 · {live_task[:10]}</option>' in page
|
|
assert f'<option value="{dead_task}">1 · {dead_task[:10]}</option>' in page
|
|
assert f'href="/plugins/docker/?stack=it{tag}"' in page
|
|
assert f'<option value="{dead_task}" selected>' in got["focus"][1]
|
|
assert got["missing"][0] == 404 and "Service not found" in got["missing"][1]
|
|
|
|
logs = got["logs"][1]
|
|
assert f"serving {tag}" in logs and f"panic {tag}" in logs # the removed task's lines too
|
|
assert logs.index(f"serving {tag}") < logs.index(f"panic {tag}") # newest first
|
|
assert f"1 · {dead_task[:10]}" in logs
|
|
one = got["one"][1]
|
|
assert f"panic {tag}" in one and f"serving {tag}" not in one
|
|
|
|
history = got["history"][1]
|
|
assert "<polyline" in history and "peak 2.0 cores" in history and "now · peak" in history
|
|
|
|
rows = got["rows"][1]
|
|
assert f'href="/plugins/docker/service/{name}"' in rows
|
|
# A stack chip keeps the search it was clicked under.
|
|
assert f'<a class="chip" href="/plugins/docker/?q=it{tag}&stack=it{tag}"' in rows
|
|
assert f'/plugins/docker/service/{name}?task={dead_task}#logs' in rows # the failure's logs
|
|
# Another stack's filter leaves it out of the table; the failures list is never filtered.
|
|
assert f'<td class="cell-name"><a href="/plugins/docker/service/{name}"' in rows
|
|
assert f'<td class="cell-name"><a href="/plugins/docker/service/{name}"' not in got["other"][1]
|