fix(docker): a container's memory is its processes' own; file cache shown on its page (#6030)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m29s
CI / integration (push) Successful in 1m45s
CI / publish (push) Successful in 1m43s

The agent sent Docker's working set (usage less inactive_file), which still
counts recently used file cache. A file-heavy container with no limit read
191 GB while its processes held 2.2 GB, and the host's own figure, which
already leaves cache out, disagreed with it.

Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so
Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes
apart. The new names keep the two meanings apart in history and in learned
normals: docker_013 renames the columns, clears the old values and adds
mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers,
container detail, normals and the MCP tools all read the new figure. File
cache is shown only on the container's page and in get_container.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 21:27:29 -04:00
co-authored by Claude Opus 5.5
parent 1335b9afaf
commit 56063f27a4
26 changed files with 226 additions and 99 deletions
+3 -3
View File
@@ -101,7 +101,7 @@ def test_a_monitor_down_stays_down_whatever_its_normal():
def test_a_containers_cpu_is_learned_per_core_and_said_in_cores():
# 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual.
c = NS(cpu_pct=380.0, mem_pct=50.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
c = NS(cpu_pct=380.0, mem_used_pct=50.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30)
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)})
assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour"
assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit"
@@ -109,8 +109,8 @@ def test_a_containers_cpu_is_learned_per_core_and_said_in_cores():
def test_a_containers_memory_within_its_normal_reads_healthy():
c = NS(cpu_pct=1.0, mem_pct=92.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_pct": _norm(90.0)})
c = NS(cpu_pct=1.0, mem_used_pct=92.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30)
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_used_pct": _norm(90.0)})
assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour")
assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit"
+24 -14
View File
@@ -136,8 +136,9 @@ def test_persist_scopes_containers_by_host(app):
now = datetime.now(timezone.utc)
snapshot = [(now, [{
"name": "web", "container_id": "abc123", "image": "nginx",
"status": "running", "cpu_pct": 12.5, "mem_pct": 30.0,
"mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None,
"status": "running", "cpu_pct": 12.5, "mem_used_pct": 30.0,
"mem_used_bytes": 100, "mem_limit_bytes": 200, "mem_cache_bytes": 900,
"ports": [], "started_at": None,
# enrichment (agent ≥ 1.4.0)
"health": "healthy", "restart_count": 2, "exit_code": 0, "oom_killed": False,
"compose_project": "stack", "service_name": "web",
@@ -165,7 +166,7 @@ def test_persist_scopes_containers_by_host(app):
# at the snapshot's time.
metric_hosts = (await s.execute(text(
"SELECT COUNT(DISTINCT resource_name) FROM plugin_metrics "
"WHERE source_module = 'docker' AND metric_name = 'mem_usage_bytes' "
"WHERE source_module = 'docker' AND metric_name = 'mem_used_bytes' "
"AND recorded_at = :at"), {"at": now})).scalar()
resources = {r[0] for r in (await s.execute(text(
"SELECT DISTINCT resource_name FROM plugin_metrics "
@@ -174,13 +175,22 @@ def test_persist_scopes_containers_by_host(app):
enrich = (await s.execute(text(
"SELECT health, restart_count, service_name, net_rx_bytes "
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
return web_rows, metric_hosts, resources, enrich
# File cache is kept beside the memory figure, not in it (#6030).
memory = (await s.execute(text(
"SELECT mem_used_bytes, mem_used_pct, mem_cache_bytes "
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
cache_series = (await s.execute(text(
"SELECT COUNT(*) FROM plugin_metrics WHERE source_module = 'docker' "
"AND metric_name = 'mem_cache_bytes' AND value = 900"))).scalar()
return web_rows, metric_hosts, resources, enrich, memory, cache_series
web_rows, metric_hosts, resources, enrich = asyncio.run(_go())
web_rows, metric_hosts, resources, enrich, memory, cache_series = asyncio.run(_go())
assert web_rows == 2 # one 'web' per host, keyed by (host_id, name)
assert metric_hosts == 2 # time-series rows scoped per host
assert resources == {"alpha/web", "beta/web"}
assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips
assert memory == (100, 30.0, 900)
assert cache_series == 2
@_NEEDS_DB
@@ -325,7 +335,7 @@ def test_persist_logs_respects_toggle_and_exclude(app):
@_NEEDS_DB
def test_large_memory_values_persist_as_bigint(app):
"""A container using >2^31 bytes of RAM must persist. Regression: mem_usage_bytes
"""A container using >2^31 bytes of RAM must persist. Regression: mem_used_bytes
/ mem_limit_bytes were int4 and overflowed (asyncpg 'value out of int32 range'),
failing the whole ingest batch for any host with a >2.1 GB container."""
from sqlalchemy import text
@@ -337,8 +347,8 @@ def test_large_memory_values_persist_as_bigint(app):
big_limit = 17_179_869_184 # 16 GB
snapshot = [(now, [{
"name": "bigmem", "container_id": "deadbeef", "image": "postgres",
"status": "running", "cpu_pct": 1.0, "mem_pct": 50.0,
"mem_usage_bytes": big_usage, "mem_limit_bytes": big_limit,
"status": "running", "cpu_pct": 1.0, "mem_used_pct": 50.0,
"mem_used_bytes": big_usage, "mem_limit_bytes": big_limit,
"ports": [], "started_at": None,
}])]
@@ -351,12 +361,12 @@ def test_large_memory_values_persist_as_bigint(app):
await s.flush()
await persist(s, h, snapshot)
row = (await s.execute(text(
"SELECT mem_usage_bytes, mem_limit_bytes FROM docker_containers "
"SELECT mem_used_bytes, mem_limit_bytes FROM docker_containers "
"WHERE name = 'bigmem'"))).first()
metric = (await s.execute(text(
"SELECT value FROM plugin_metrics WHERE source_module = 'docker' "
"AND resource_name = 'bigmemhost/bigmem' "
"AND metric_name = 'mem_usage_bytes'"))).scalar()
"AND metric_name = 'mem_used_bytes'"))).scalar()
return row, metric
row, metric = asyncio.run(_go())
@@ -372,7 +382,7 @@ def test_lifecycle_events_derived_across_snapshots(app):
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
running = {"name": "web", "container_id": "abc", "image": "nginx",
"status": "running", "cpu_pct": 5.0, "mem_pct": 10.0,
"status": "running", "cpu_pct": 5.0, "mem_used_pct": 10.0,
"restart_count": 0, "exit_code": None, "oom_killed": False,
"health": "healthy"}
died = {**running, "status": "exited", "cpu_pct": None,
@@ -414,7 +424,7 @@ def test_vanished_container_is_reaped(app):
persist = _persist_fn(app)
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
base = {"status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
base = {"status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
keep = {**base, "name": "keep"}
gone = {**base, "name": "ci-job-123"}
@@ -456,7 +466,7 @@ def test_persist_marks_each_part_for_the_slow_ingest_line(app):
persist = _persist_fn(app)
at = datetime(2026, 10, 6, 23, 0, 0, tzinfo=timezone.utc)
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
logs = [(at, [{"container": "web", "stream": "stdout", "ts": None, "line": "hello"}])]
marks = _Marks()
@@ -486,7 +496,7 @@ def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
persist = _persist_fn(app)
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
async def _go():
@@ -47,7 +47,7 @@ def test_recent_usage_averages_each_containers_bucket(app):
when = at + timedelta(seconds=secs)
return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{name}",
metric_name=metric, value=value, recorded_at=when)
for metric, value in (("cpu_pct", cpu), ("mem_usage_bytes", mem))]
for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))]
async def go():
async with app.db_sessionmaker() as s:
@@ -61,7 +61,7 @@ def test_a_service_page_reads_every_task(app):
when = now - timedelta(minutes=mins)
return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{cname}",
metric_name=metric, value=value, recorded_at=when)
for metric, value in (("cpu_pct", cpu), ("mem_usage_bytes", mem))]
for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))]
async def go():
async with app.db_sessionmaker() as s:
@@ -78,7 +78,7 @@ def test_a_service_page_reads_every_task(app):
placement_json=json.dumps([{"node_id": node, "running": 1}]),
tasks_json=json.dumps(tasks), updated_at=now),
m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running",
cpu_pct=50.0, mem_usage_bytes=100 << 20, service_name=name,
cpu_pct=50.0, mem_used_bytes=100 << 20, service_name=name,
task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9),
scraped_at=now),
# Docker has removed the failed task's container: only its events are left.
@@ -30,7 +30,7 @@ def test_ingest_records_docker_status_and_clears_on_an_empty_answer(app):
from sqlalchemy import text
from steward.models.hosts import Host
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
async def _go():
+10 -6
View File
@@ -70,10 +70,10 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
container_sql = text("""
INSERT INTO docker_containers
(host_id, name, container_id, image, status, cpu_pct, mem_pct, mem_usage_bytes,
mem_limit_bytes, restart_count, ports_json, started_at, scraped_at, health,
exit_code, oom_killed, compose_project, service_name, task_id)
VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, :rc, '[]', :started, :now,
(host_id, name, container_id, image, status, cpu_pct, mem_used_pct, mem_used_bytes,
mem_limit_bytes, mem_cache_bytes, restart_count, ports_json, started_at, scraped_at,
health, exit_code, oom_killed, compose_project, service_name, task_id)
VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, 5000, :rc, '[]', :started, :now,
:health, :exit, false, :proj, :svc, :task)
""")
@@ -134,8 +134,8 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
"ts": now - timedelta(minutes=5 - i), "st": stream, "line": line})
# Container series live in the one metric store (#5752).
for i, cpu in enumerate((10.0, 30.0, 50.0)):
for metric, value in (("cpu_pct", cpu), ("mem_pct", 20.0),
("mem_usage_bytes", 1000.0)):
for metric, value in (("cpu_pct", cpu), ("mem_used_pct", 20.0),
("mem_used_bytes", 1000.0)):
await s.execute(text(
"INSERT INTO plugin_metrics (id, source_module, resource_name, "
"metric_name, value, recorded_at) "
@@ -201,6 +201,10 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
c = await dtools.get_container(web, range="1h")
assert c["host_name"] == host_name and c["status"] == "running"
assert c["cpu_pct_history"]["max"] == 50.0 and c["cpu_pct_history"]["points"] == 3
assert c["mem_used_pct_history"]["max"] == 20.0
# File cache is in the one container's detail, not in the list rows (#6030).
assert c["mem_used_bytes"] == 1000 and c["mem_cache_bytes"] == 5000
assert "mem_cache_bytes" not in proj[0]
crashed = await dtools.get_container(job, host=host_name)
assert [e["event"] for e in crashed["events"]] == ["die"]
# A unique part of a long swarm task name is enough.
+1 -1
View File
@@ -16,7 +16,7 @@ NOW = datetime.now(timezone.utc)
def _c(name, **kw):
base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None,
oom_killed=False, restart_count=0, cpu_pct=5.0, mem_pct=10.0, compose_project=None,
oom_killed=False, restart_count=0, cpu_pct=5.0, mem_used_pct=10.0, compose_project=None,
service_name=None, ports_json=None, started_at=NOW - timedelta(hours=2),
scraped_at=NOW - timedelta(seconds=20))
base.update(kw)
+2 -2
View File
@@ -15,8 +15,8 @@ NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
def _c(**kw):
base = dict(name="web", host_id="h1", image="nginx", status="running", health=None,
exit_code=None, oom_killed=False, restart_count=0, cpu_pct=1.0, mem_pct=2.0,
mem_usage_bytes=1, mem_limit_bytes=2, started_at=NOW - timedelta(hours=1),
exit_code=None, oom_killed=False, restart_count=0, cpu_pct=1.0, mem_used_pct=2.0,
mem_used_bytes=1, mem_limit_bytes=2, started_at=NOW - timedelta(hours=1),
compose_project=None, service_name=None, scraped_at=NOW - timedelta(seconds=30))
base.update(kw)
return NS(**base)
+14 -2
View File
@@ -265,7 +265,7 @@ def _ctr(**kw):
base = dict(name="plex", host_id="h1", image="plexinc/pms:1.40", status="exited", health=None,
exit_code=137, oom_killed=True, restart_count=2, compose_project="media",
service_name=None, node_id=None, scraped_at=NOW - timedelta(seconds=30),
started_at=NOW - timedelta(hours=3))
started_at=NOW - timedelta(hours=3), mem_cache_bytes=None)
base.update(kw)
return NS(**base)
@@ -312,6 +312,18 @@ async def test_a_running_swarm_task_reads_under_its_service(app):
assert "<dt>Swarm node</dt>" in page and "docker02" in page
async def test_file_cache_is_told_on_the_container_page_apart_from_its_memory(app):
"""#6030: its memory figure is what its processes hold; the file cache
charged to it is shown here, said to be given back, and is "—" while
nothing reports it."""
running = _ctr(status="running", exit_code=0, oom_killed=False, mem_cache_bytes=180 << 30)
page = await _container_page(app, running, exit_failed=False, uptime="3h")
assert "<dt>File cache</dt><dd>180.0 GiB" in page
assert "kept in memory until the host needs it back; not counted in its memory" in page
stopped = await _container_page(app, _ctr())
assert '<dt>File cache</dt><dd><span class="tone-dim">—</span></dd>' in stopped
async def test_a_service_log_line_names_its_task(app):
lines = [{"ts": NOW, "stream": "stderr", "line": "<b>boom</b>", "task": f"1 · {TASK[:10]}"}]
async with app.test_request_context("/plugins/docker/service/web/logs/lines"):
@@ -335,7 +347,7 @@ def _swarm_svc(name, placement):
def _task(name, service, node, cpu, mem, status="running"):
return NS(name=name, service_name=service, node_id=node, status=status, cpu_pct=cpu,
mem_usage_bytes=mem)
mem_used_bytes=mem)
def test_each_node_lists_what_runs_on_it_and_what_that_uses():
+2 -2
View File
@@ -81,12 +81,12 @@ def test_task_label_names_the_service_and_slot():
def test_usage_reads_cpu_in_cores_and_memory_against_a_real_limit():
gib = 1024 ** 3
c = NS(cpu_pct=150.0, mem_usage_bytes=gib, mem_limit_bytes=16 * gib, mem_pct=6.25)
c = NS(cpu_pct=150.0, mem_used_bytes=gib, mem_limit_bytes=16 * gib, mem_used_pct=6.25)
u = svc.usage(c, 4, 16 * gib)
assert u["cores"] == 1.5 and u["cpu_share"] == 37.5
# A limit of the host's whole memory is Docker saying "no limit".
assert u["mem_limit"] is None and u["mem_of"] == "the host's memory"
limited = svc.usage(NS(cpu_pct=None, mem_usage_bytes=gib, mem_limit_bytes=2 * gib, mem_pct=50.0), 4, 16 * gib)
limited = svc.usage(NS(cpu_pct=None, mem_used_bytes=gib, mem_limit_bytes=2 * gib, mem_used_pct=50.0), 4, 16 * gib)
assert limited["cores"] is None and limited["mem_limit"] == 2 * gib and limited["mem_of"] == "its limit"
# Unknown host size: cores still read, no share to grade against.
assert svc.usage(c, None, None)["cpu_share"] is None
+1 -1
View File
@@ -17,7 +17,7 @@ NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc)
def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0,
task=None, started=None):
return SimpleNamespace(host_id=host_id, name=name, service_name=service,
node_id=node, status=status, cpu_pct=cpu, mem_pct=2.0,
node_id=node, status=status, cpu_pct=cpu, mem_used_pct=2.0,
health=None, restart_count=0, container_id=name,
task_id=task, started_at=started)
+37 -10
View File
@@ -50,16 +50,41 @@ def test_docker_cpu_pct_guards_bad_stats():
}) == 0.0
def test_docker_mem_subtracts_cache():
def test_docker_mem_is_process_memory_not_file_cache():
# cgroup v2, as curator_app read on 2026-10-17 (#6030): 315 GB charged to
# it, 2.2 GB its own; Docker's working set (usage − inactive_file) said 191 GB.
stats = {"memory_stats": {
"usage": 200 * 1024 * 1024,
"limit": 1000 * 1024 * 1024,
"stats": {"cache": 50 * 1024 * 1024},
"usage": 314_896_379_904,
"limit": 405_493_768_192,
"stats": {"anon": 2_241_888_256, "shmem": 106_496, "file": 312_003_436_544,
"active_file": 188_779_880_448, "inactive_file": 123_223_449_600},
}}
usage, limit, pct = a._docker_mem(stats)
assert usage == 150 * 1024 * 1024 # working set = usage − cache
assert limit == 1000 * 1024 * 1024
assert pct == 15.0
used, limit, pct, cache = a._docker_mem(stats)
assert used == 2_241_888_256 + 106_496
assert limit == 405_493_768_192
assert pct == 0.55
assert cache == 312_003_436_544 - 106_496 # shmem is in file; counted as used, not cache
def test_docker_mem_counts_shared_memory_as_used():
# Postgres keeps its shared buffers in shmem: not anon, but not reclaimable.
mib = 1024 * 1024
stats = {"memory_stats": {"usage": 900 * mib, "limit": 1000 * mib,
"stats": {"anon": 100 * mib, "shmem": 400 * mib, "file": 700 * mib}}}
assert a._docker_mem(stats) == (500 * mib, 1000 * mib, 50.0, 300 * mib)
def test_docker_mem_reads_cgroup_v1():
# v1's rss leaves shmem out and its cache has it in.
mib = 1024 * 1024
stats = {"memory_stats": {"usage": 200 * mib, "limit": 1000 * mib,
"stats": {"total_rss": 120 * mib, "total_shmem": 30 * mib,
"total_cache": 80 * mib, "rss": 1, "cache": 1}}}
assert a._docker_mem(stats) == (150 * mib, 1000 * mib, 15.0, 50 * mib)
def test_docker_mem_without_stats_is_zero():
assert a._docker_mem({}) == (0, 0, 0.0, 0)
def test_docker_ports_keeps_only_published():
@@ -162,7 +187,8 @@ def test_collect_docker_assembles_container(monkeypatch):
"precpu_stats": {"cpu_usage": {"total_usage": 100_000_000},
"system_cpu_usage": 1_000_000_000},
"memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024,
"stats": {"cache": 50 * 1024 * 1024}},
"stats": {"anon": 140 * 1024 * 1024, "shmem": 10 * 1024 * 1024,
"file": 60 * 1024 * 1024}},
"networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}},
"blkio_stats": {"io_service_bytes_recursive": [
{"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]},
@@ -187,7 +213,8 @@ def test_collect_docker_assembles_container(monkeypatch):
assert c["image"] == "nginx:latest"
assert c["status"] == "running"
assert c["cpu_pct"] == 40.0
assert c["mem_pct"] == 15.0
assert c["mem_used_bytes"] == 150 * 1024 * 1024 and c["mem_used_pct"] == 15.0
assert c["mem_cache_bytes"] == 50 * 1024 * 1024
assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
# When it last started (inspect), not when it was created (2020).
assert c["started_at"] == "2023-11-14T22:13:20.123456+00:00"