fix(docker): a container's memory is its processes' own; file cache shown on its page (#6030)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m29s
CI / integration (push) Successful in 1m45s
CI / publish (push) Successful in 1m43s

The agent sent Docker's working set (usage less inactive_file), which still
counts recently used file cache. A file-heavy container with no limit read
191 GB while its processes held 2.2 GB, and the host's own figure, which
already leaves cache out, disagreed with it.

Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so
Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes
apart. The new names keep the two meanings apart in history and in learned
normals: docker_013 renames the columns, clears the old values and adds
mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers,
container detail, normals and the MCP tools all read the new figure. File
cache is shown only on the container's page and in get_container.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 21:27:29 -04:00
co-authored by Claude Opus 5.5
parent 1335b9afaf
commit 56063f27a4
26 changed files with 226 additions and 99 deletions
+24 -14
View File
@@ -136,8 +136,9 @@ def test_persist_scopes_containers_by_host(app):
now = datetime.now(timezone.utc)
snapshot = [(now, [{
"name": "web", "container_id": "abc123", "image": "nginx",
"status": "running", "cpu_pct": 12.5, "mem_pct": 30.0,
"mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None,
"status": "running", "cpu_pct": 12.5, "mem_used_pct": 30.0,
"mem_used_bytes": 100, "mem_limit_bytes": 200, "mem_cache_bytes": 900,
"ports": [], "started_at": None,
# enrichment (agent ≥ 1.4.0)
"health": "healthy", "restart_count": 2, "exit_code": 0, "oom_killed": False,
"compose_project": "stack", "service_name": "web",
@@ -165,7 +166,7 @@ def test_persist_scopes_containers_by_host(app):
# at the snapshot's time.
metric_hosts = (await s.execute(text(
"SELECT COUNT(DISTINCT resource_name) FROM plugin_metrics "
"WHERE source_module = 'docker' AND metric_name = 'mem_usage_bytes' "
"WHERE source_module = 'docker' AND metric_name = 'mem_used_bytes' "
"AND recorded_at = :at"), {"at": now})).scalar()
resources = {r[0] for r in (await s.execute(text(
"SELECT DISTINCT resource_name FROM plugin_metrics "
@@ -174,13 +175,22 @@ def test_persist_scopes_containers_by_host(app):
enrich = (await s.execute(text(
"SELECT health, restart_count, service_name, net_rx_bytes "
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
return web_rows, metric_hosts, resources, enrich
# File cache is kept beside the memory figure, not in it (#6030).
memory = (await s.execute(text(
"SELECT mem_used_bytes, mem_used_pct, mem_cache_bytes "
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
cache_series = (await s.execute(text(
"SELECT COUNT(*) FROM plugin_metrics WHERE source_module = 'docker' "
"AND metric_name = 'mem_cache_bytes' AND value = 900"))).scalar()
return web_rows, metric_hosts, resources, enrich, memory, cache_series
web_rows, metric_hosts, resources, enrich = asyncio.run(_go())
web_rows, metric_hosts, resources, enrich, memory, cache_series = asyncio.run(_go())
assert web_rows == 2 # one 'web' per host, keyed by (host_id, name)
assert metric_hosts == 2 # time-series rows scoped per host
assert resources == {"alpha/web", "beta/web"}
assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips
assert memory == (100, 30.0, 900)
assert cache_series == 2
@_NEEDS_DB
@@ -325,7 +335,7 @@ def test_persist_logs_respects_toggle_and_exclude(app):
@_NEEDS_DB
def test_large_memory_values_persist_as_bigint(app):
"""A container using >2^31 bytes of RAM must persist. Regression: mem_usage_bytes
"""A container using >2^31 bytes of RAM must persist. Regression: mem_used_bytes
/ mem_limit_bytes were int4 and overflowed (asyncpg 'value out of int32 range'),
failing the whole ingest batch for any host with a >2.1 GB container."""
from sqlalchemy import text
@@ -337,8 +347,8 @@ def test_large_memory_values_persist_as_bigint(app):
big_limit = 17_179_869_184 # 16 GB
snapshot = [(now, [{
"name": "bigmem", "container_id": "deadbeef", "image": "postgres",
"status": "running", "cpu_pct": 1.0, "mem_pct": 50.0,
"mem_usage_bytes": big_usage, "mem_limit_bytes": big_limit,
"status": "running", "cpu_pct": 1.0, "mem_used_pct": 50.0,
"mem_used_bytes": big_usage, "mem_limit_bytes": big_limit,
"ports": [], "started_at": None,
}])]
@@ -351,12 +361,12 @@ def test_large_memory_values_persist_as_bigint(app):
await s.flush()
await persist(s, h, snapshot)
row = (await s.execute(text(
"SELECT mem_usage_bytes, mem_limit_bytes FROM docker_containers "
"SELECT mem_used_bytes, mem_limit_bytes FROM docker_containers "
"WHERE name = 'bigmem'"))).first()
metric = (await s.execute(text(
"SELECT value FROM plugin_metrics WHERE source_module = 'docker' "
"AND resource_name = 'bigmemhost/bigmem' "
"AND metric_name = 'mem_usage_bytes'"))).scalar()
"AND metric_name = 'mem_used_bytes'"))).scalar()
return row, metric
row, metric = asyncio.run(_go())
@@ -372,7 +382,7 @@ def test_lifecycle_events_derived_across_snapshots(app):
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
running = {"name": "web", "container_id": "abc", "image": "nginx",
"status": "running", "cpu_pct": 5.0, "mem_pct": 10.0,
"status": "running", "cpu_pct": 5.0, "mem_used_pct": 10.0,
"restart_count": 0, "exit_code": None, "oom_killed": False,
"health": "healthy"}
died = {**running, "status": "exited", "cpu_pct": None,
@@ -414,7 +424,7 @@ def test_vanished_container_is_reaped(app):
persist = _persist_fn(app)
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
base = {"status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
base = {"status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
keep = {**base, "name": "keep"}
gone = {**base, "name": "ci-job-123"}
@@ -456,7 +466,7 @@ def test_persist_marks_each_part_for_the_slow_ingest_line(app):
persist = _persist_fn(app)
at = datetime(2026, 10, 6, 23, 0, 0, tzinfo=timezone.utc)
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
logs = [(at, [{"container": "web", "stream": "stdout", "ts": None, "line": "hello"}])]
marks = _Marks()
@@ -486,7 +496,7 @@ def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
persist = _persist_fn(app)
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
async def _go():