fix(docker): a container's memory is its processes' own; file cache shown on its page (#6030)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m29s
CI / integration (push) Successful in 1m45s
CI / publish (push) Successful in 1m43s

The agent sent Docker's working set (usage less inactive_file), which still
counts recently used file cache. A file-heavy container with no limit read
191 GB while its processes held 2.2 GB, and the host's own figure, which
already leaves cache out, disagreed with it.

Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so
Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes
apart. The new names keep the two meanings apart in history and in learned
normals: docker_013 renames the columns, clears the old values and adds
mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers,
container detail, normals and the MCP tools all read the new figure. File
cache is shown only on the container's page and in get_container.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 21:27:29 -04:00
co-authored by Claude Opus 5.5
parent 1335b9afaf
commit 56063f27a4
26 changed files with 226 additions and 99 deletions
+18 -10
View File
@@ -62,8 +62,8 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime,
"oom_killed": bool(c.oom_killed),
"restart_count": c.restart_count,
"cpu_pct": rnd(c.cpu_pct),
"mem_pct": rnd(c.mem_pct),
"mem_usage_bytes": c.mem_usage_bytes,
"mem_used_pct": rnd(c.mem_used_pct),
"mem_used_bytes": c.mem_used_bytes,
"mem_limit_bytes": c.mem_limit_bytes,
"started_at": iso(c.started_at),
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
@@ -388,9 +388,10 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
a name also works (swarm task names are long).
host: the host's name or id. Needed only when several hosts run a
container with this name.
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
range: the window for `cpu_pct_history` and `mem_used_pct_history` (min,
avg, max, p95, last and a ≤60-point series) — "1h", "6h", "24h"
(default), "7d" or "30d". `cpu_pct` and `mem_used_pct` are the latest
reported values.
events: how many lifecycle events (start, restart, stop, die, oom, health_change)
to return, newest first (default 20, max 100).
@@ -399,6 +400,12 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
removed container, while Steward retains its logs or events: the reply
has `removed: true`, `last_seen_at`, its history and its events, and no
live state. A live container has `removed: false`.
Memory (`mem_used_bytes`, `mem_used_pct`, of its limit or of the host's
RAM when it has none) is what its processes hold: anon + shmem.
`mem_cache_bytes` is the file cache charged to it, which the kernel
gives back when the host needs the memory, so a large figure there is
not memory the container needs.
"""
since, key = lookback(range)
events = clamp_limit(events, 20, 100)
@@ -412,7 +419,7 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
host_names = await _host_names(db)
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
hour_ago = now - timedelta(hours=1)
@@ -430,9 +437,10 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
"ports": ports,
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
"mem_cache_bytes": c.mem_cache_bytes,
"range": key,
"cpu_pct_history": summarize(cpu),
"mem_pct_history": summarize(mem),
"mem_used_pct_history": summarize(mem),
"events": _event_rows(evs, now),
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
}
@@ -448,7 +456,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i
"""get_container for a container Docker has removed: what Steward kept."""
host_names = await _host_names(db)
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
return {
"container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
@@ -456,7 +464,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i
"last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
"range": key,
"cpu_pct_history": summarize(cpu),
"mem_pct_history": summarize(mem),
"mem_used_pct_history": summarize(mem),
"events": _event_rows(evs, now),
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
}
@@ -536,7 +544,7 @@ async def list_swarm_services(limit: int = 100) -> dict:
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
"cpu_pct": rnd(r["cpu_pct"]), "mem_used_pct": rnd(r["mem_pct"])}
for r in s["replicas"]
],
}