fix(docker): a container's memory is its processes' own; file cache shown on its page (#6030)
The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -136,8 +136,9 @@ def test_persist_scopes_containers_by_host(app):
|
||||
now = datetime.now(timezone.utc)
|
||||
snapshot = [(now, [{
|
||||
"name": "web", "container_id": "abc123", "image": "nginx",
|
||||
"status": "running", "cpu_pct": 12.5, "mem_pct": 30.0,
|
||||
"mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None,
|
||||
"status": "running", "cpu_pct": 12.5, "mem_used_pct": 30.0,
|
||||
"mem_used_bytes": 100, "mem_limit_bytes": 200, "mem_cache_bytes": 900,
|
||||
"ports": [], "started_at": None,
|
||||
# enrichment (agent ≥ 1.4.0)
|
||||
"health": "healthy", "restart_count": 2, "exit_code": 0, "oom_killed": False,
|
||||
"compose_project": "stack", "service_name": "web",
|
||||
@@ -165,7 +166,7 @@ def test_persist_scopes_containers_by_host(app):
|
||||
# at the snapshot's time.
|
||||
metric_hosts = (await s.execute(text(
|
||||
"SELECT COUNT(DISTINCT resource_name) FROM plugin_metrics "
|
||||
"WHERE source_module = 'docker' AND metric_name = 'mem_usage_bytes' "
|
||||
"WHERE source_module = 'docker' AND metric_name = 'mem_used_bytes' "
|
||||
"AND recorded_at = :at"), {"at": now})).scalar()
|
||||
resources = {r[0] for r in (await s.execute(text(
|
||||
"SELECT DISTINCT resource_name FROM plugin_metrics "
|
||||
@@ -174,13 +175,22 @@ def test_persist_scopes_containers_by_host(app):
|
||||
enrich = (await s.execute(text(
|
||||
"SELECT health, restart_count, service_name, net_rx_bytes "
|
||||
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
|
||||
return web_rows, metric_hosts, resources, enrich
|
||||
# File cache is kept beside the memory figure, not in it (#6030).
|
||||
memory = (await s.execute(text(
|
||||
"SELECT mem_used_bytes, mem_used_pct, mem_cache_bytes "
|
||||
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
|
||||
cache_series = (await s.execute(text(
|
||||
"SELECT COUNT(*) FROM plugin_metrics WHERE source_module = 'docker' "
|
||||
"AND metric_name = 'mem_cache_bytes' AND value = 900"))).scalar()
|
||||
return web_rows, metric_hosts, resources, enrich, memory, cache_series
|
||||
|
||||
web_rows, metric_hosts, resources, enrich = asyncio.run(_go())
|
||||
web_rows, metric_hosts, resources, enrich, memory, cache_series = asyncio.run(_go())
|
||||
assert web_rows == 2 # one 'web' per host, keyed by (host_id, name)
|
||||
assert metric_hosts == 2 # time-series rows scoped per host
|
||||
assert resources == {"alpha/web", "beta/web"}
|
||||
assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips
|
||||
assert memory == (100, 30.0, 900)
|
||||
assert cache_series == 2
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
@@ -325,7 +335,7 @@ def test_persist_logs_respects_toggle_and_exclude(app):
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_large_memory_values_persist_as_bigint(app):
|
||||
"""A container using >2^31 bytes of RAM must persist. Regression: mem_usage_bytes
|
||||
"""A container using >2^31 bytes of RAM must persist. Regression: mem_used_bytes
|
||||
/ mem_limit_bytes were int4 and overflowed (asyncpg 'value out of int32 range'),
|
||||
failing the whole ingest batch for any host with a >2.1 GB container."""
|
||||
from sqlalchemy import text
|
||||
@@ -337,8 +347,8 @@ def test_large_memory_values_persist_as_bigint(app):
|
||||
big_limit = 17_179_869_184 # 16 GB
|
||||
snapshot = [(now, [{
|
||||
"name": "bigmem", "container_id": "deadbeef", "image": "postgres",
|
||||
"status": "running", "cpu_pct": 1.0, "mem_pct": 50.0,
|
||||
"mem_usage_bytes": big_usage, "mem_limit_bytes": big_limit,
|
||||
"status": "running", "cpu_pct": 1.0, "mem_used_pct": 50.0,
|
||||
"mem_used_bytes": big_usage, "mem_limit_bytes": big_limit,
|
||||
"ports": [], "started_at": None,
|
||||
}])]
|
||||
|
||||
@@ -351,12 +361,12 @@ def test_large_memory_values_persist_as_bigint(app):
|
||||
await s.flush()
|
||||
await persist(s, h, snapshot)
|
||||
row = (await s.execute(text(
|
||||
"SELECT mem_usage_bytes, mem_limit_bytes FROM docker_containers "
|
||||
"SELECT mem_used_bytes, mem_limit_bytes FROM docker_containers "
|
||||
"WHERE name = 'bigmem'"))).first()
|
||||
metric = (await s.execute(text(
|
||||
"SELECT value FROM plugin_metrics WHERE source_module = 'docker' "
|
||||
"AND resource_name = 'bigmemhost/bigmem' "
|
||||
"AND metric_name = 'mem_usage_bytes'"))).scalar()
|
||||
"AND metric_name = 'mem_used_bytes'"))).scalar()
|
||||
return row, metric
|
||||
|
||||
row, metric = asyncio.run(_go())
|
||||
@@ -372,7 +382,7 @@ def test_lifecycle_events_derived_across_snapshots(app):
|
||||
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
|
||||
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
|
||||
running = {"name": "web", "container_id": "abc", "image": "nginx",
|
||||
"status": "running", "cpu_pct": 5.0, "mem_pct": 10.0,
|
||||
"status": "running", "cpu_pct": 5.0, "mem_used_pct": 10.0,
|
||||
"restart_count": 0, "exit_code": None, "oom_killed": False,
|
||||
"health": "healthy"}
|
||||
died = {**running, "status": "exited", "cpu_pct": None,
|
||||
@@ -414,7 +424,7 @@ def test_vanished_container_is_reaped(app):
|
||||
persist = _persist_fn(app)
|
||||
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
|
||||
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
|
||||
base = {"status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
|
||||
base = {"status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
|
||||
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
|
||||
keep = {**base, "name": "keep"}
|
||||
gone = {**base, "name": "ci-job-123"}
|
||||
@@ -456,7 +466,7 @@ def test_persist_marks_each_part_for_the_slow_ingest_line(app):
|
||||
|
||||
persist = _persist_fn(app)
|
||||
at = datetime(2026, 10, 6, 23, 0, 0, tzinfo=timezone.utc)
|
||||
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
|
||||
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
|
||||
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
|
||||
logs = [(at, [{"container": "web", "stream": "stdout", "ts": None, "line": "hello"}])]
|
||||
marks = _Marks()
|
||||
@@ -486,7 +496,7 @@ def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
|
||||
persist = _persist_fn(app)
|
||||
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
|
||||
t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
|
||||
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
|
||||
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
|
||||
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
|
||||
|
||||
async def _go():
|
||||
|
||||
Reference in New Issue
Block a user