bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(docker): a container's memory is its processes' own; file cache shown on its page (#6030)
CI / lint (push) Successful in 2s
Details
CI / unit (push) Successful in 1m29s
Details
CI / integration (push) Successful in 1m45s
Details
CI / publish (push) Successful in 1m43s
Details

Browse Source 
The agent sent Docker's working set (usage less inactive_file), which still
counts recently used file cache. A file-heavy container with no limit read
191 GB while its processes held 2.2 GB, and the host's own figure, which
already leaves cache out, disagreed with it.

Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so
Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes
apart. The new names keep the two meanings apart in history and in learned
normals: docker_013 renames the columns, clears the old values and adds
mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers,
container detail, normals and the MCP tools all read the new figure. File
cache is shown only on the container's page and in get_container.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-17 21:27:29 -04:00
co-authored by Claude Opus 5.5
parent 1335b9afaf
commit 56063f27a4
26 changed files with 226 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/test_docker.py
+24 -14
View File
@@ -136,8 +136,9 @@ def test_persist_scopes_containers_by_host(app):
now = datetime.now(timezone.utc)
snapshot = [(now, [{
"name": "web", "container_id": "abc123", "image": "nginx",
"status": "running", "cpu_pct": 12.5, "mem_pct": 30.0,
"mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None,
"status": "running", "cpu_pct": 12.5, "mem_used_pct": 30.0,
"mem_used_bytes": 100, "mem_limit_bytes": 200, "mem_cache_bytes": 900,
"ports": [], "started_at": None,
# enrichment (agent ≥ 1.4.0)
"health": "healthy", "restart_count": 2, "exit_code": 0, "oom_killed": False,
"compose_project": "stack", "service_name": "web",
@@ -165,7 +166,7 @@ def test_persist_scopes_containers_by_host(app):
# at the snapshot's time.
metric_hosts = (await s.execute(text(
"SELECT COUNT(DISTINCT resource_name) FROM plugin_metrics "
"WHERE source_module = 'docker' AND metric_name = 'mem_usage_bytes' "
"WHERE source_module = 'docker' AND metric_name = 'mem_used_bytes' "
"AND recorded_at = :at"), {"at": now})).scalar()
resources = {r[0] for r in (await s.execute(text(
"SELECT DISTINCT resource_name FROM plugin_metrics "
@@ -174,13 +175,22 @@ def test_persist_scopes_containers_by_host(app):
enrich = (await s.execute(text(
"SELECT health, restart_count, service_name, net_rx_bytes "
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
return web_rows, metric_hosts, resources, enrich
# File cache is kept beside the memory figure, not in it (#6030).
memory = (await s.execute(text(
"SELECT mem_used_bytes, mem_used_pct, mem_cache_bytes "
"FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
cache_series = (await s.execute(text(
"SELECT COUNT(*) FROM plugin_metrics WHERE source_module = 'docker' "
"AND metric_name = 'mem_cache_bytes' AND value = 900"))).scalar()
return web_rows, metric_hosts, resources, enrich, memory, cache_series
web_rows, metric_hosts, resources, enrich = asyncio.run(_go())
web_rows, metric_hosts, resources, enrich, memory, cache_series = asyncio.run(_go())
assert web_rows == 2 # one 'web' per host, keyed by (host_id, name)
assert metric_hosts == 2 # time-series rows scoped per host
assert resources == {"alpha/web", "beta/web"}
assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips
assert memory == (100, 30.0, 900)
assert cache_series == 2
@_NEEDS_DB
@@ -325,7 +335,7 @@ def test_persist_logs_respects_toggle_and_exclude(app):
@_NEEDS_DB
def test_large_memory_values_persist_as_bigint(app):
"""A container using >2^31 bytes of RAM must persist. Regression: mem_usage_bytes
"""A container using >2^31 bytes of RAM must persist. Regression: mem_used_bytes
/ mem_limit_bytes were int4 and overflowed (asyncpg 'value out of int32 range'),
failing the whole ingest batch for any host with a >2.1 GB container."""
from sqlalchemy import text
@@ -337,8 +347,8 @@ def test_large_memory_values_persist_as_bigint(app):
big_limit = 17_179_869_184 # 16 GB
snapshot = [(now, [{
"name": "bigmem", "container_id": "deadbeef", "image": "postgres",
"status": "running", "cpu_pct": 1.0, "mem_pct": 50.0,
"mem_usage_bytes": big_usage, "mem_limit_bytes": big_limit,
"status": "running", "cpu_pct": 1.0, "mem_used_pct": 50.0,
"mem_used_bytes": big_usage, "mem_limit_bytes": big_limit,
"ports": [], "started_at": None,
}])]
@@ -351,12 +361,12 @@ def test_large_memory_values_persist_as_bigint(app):
await s.flush()
await persist(s, h, snapshot)
row = (await s.execute(text(
"SELECT mem_usage_bytes, mem_limit_bytes FROM docker_containers "
"SELECT mem_used_bytes, mem_limit_bytes FROM docker_containers "
"WHERE name = 'bigmem'"))).first()
metric = (await s.execute(text(
"SELECT value FROM plugin_metrics WHERE source_module = 'docker' "
"AND resource_name = 'bigmemhost/bigmem' "
"AND metric_name = 'mem_usage_bytes'"))).scalar()
"AND metric_name = 'mem_used_bytes'"))).scalar()
return row, metric
row, metric = asyncio.run(_go())
@@ -372,7 +382,7 @@ def test_lifecycle_events_derived_across_snapshots(app):
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
running = {"name": "web", "container_id": "abc", "image": "nginx",
"status": "running", "cpu_pct": 5.0, "mem_pct": 10.0,
"status": "running", "cpu_pct": 5.0, "mem_used_pct": 10.0,
"restart_count": 0, "exit_code": None, "oom_killed": False,
"health": "healthy"}
died = {**running, "status": "exited", "cpu_pct": None,
@@ -414,7 +424,7 @@ def test_vanished_container_is_reaped(app):
persist = _persist_fn(app)
t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
base = {"status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
base = {"status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
keep = {**base, "name": "keep"}
gone = {**base, "name": "ci-job-123"}
@@ -456,7 +466,7 @@ def test_persist_marks_each_part_for_the_slow_ingest_line(app):
persist = _persist_fn(app)
at = datetime(2026, 10, 6, 23, 0, 0, tzinfo=timezone.utc)
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
logs = [(at, [{"container": "web", "stream": "stdout", "ts": None, "line": "hello"}])]
marks = _Marks()
@@ -486,7 +496,7 @@ def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
persist = _persist_fn(app)
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
async def _go():
tests/integration/test_docker_recent_usage.py
+1 -1
View File
@@ -47,7 +47,7 @@ def test_recent_usage_averages_each_containers_bucket(app):
when = at + timedelta(seconds=secs)
return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{name}",
metric_name=metric, value=value, recorded_at=when)
for metric, value in (("cpu_pct", cpu), ("mem_usage_bytes", mem))]
for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))]
async def go():
async with app.db_sessionmaker() as s:
tests/integration/test_docker_service_page.py
+2 -2
View File
@@ -61,7 +61,7 @@ def test_a_service_page_reads_every_task(app):
when = now - timedelta(minutes=mins)
return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{cname}",
metric_name=metric, value=value, recorded_at=when)
for metric, value in (("cpu_pct", cpu), ("mem_usage_bytes", mem))]
for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))]
async def go():
async with app.db_sessionmaker() as s:
@@ -78,7 +78,7 @@ def test_a_service_page_reads_every_task(app):
placement_json=json.dumps([{"node_id": node, "running": 1}]),
tasks_json=json.dumps(tasks), updated_at=now),
m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running",
cpu_pct=50.0, mem_usage_bytes=100 << 20, service_name=name,
cpu_pct=50.0, mem_used_bytes=100 << 20, service_name=name,
task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9),
scraped_at=now),
# Docker has removed the failed task's container: only its events are left.
tests/integration/test_host_agent_docker_status.py
+1 -1
View File
@@ -30,7 +30,7 @@ def test_ingest_records_docker_status_and_clears_on_an_empty_answer(app):
from sqlalchemy import text
from steward.models.hosts import Host
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
async def _go():
tests/integration/test_mcp_plugin_tools.py
+10 -6
View File
@@ -70,10 +70,10 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
container_sql = text("""
INSERT INTO docker_containers
(host_id, name, container_id, image, status, cpu_pct, mem_pct, mem_usage_bytes,
mem_limit_bytes, restart_count, ports_json, started_at, scraped_at, health,
exit_code, oom_killed, compose_project, service_name, task_id)
VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, :rc, '[]', :started, :now,
(host_id, name, container_id, image, status, cpu_pct, mem_used_pct, mem_used_bytes,
mem_limit_bytes, mem_cache_bytes, restart_count, ports_json, started_at, scraped_at,
health, exit_code, oom_killed, compose_project, service_name, task_id)
VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, 5000, :rc, '[]', :started, :now,
:health, :exit, false, :proj, :svc, :task)
""")
@@ -134,8 +134,8 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
"ts": now - timedelta(minutes=5 - i), "st": stream, "line": line})
# Container series live in the one metric store (#5752).
for i, cpu in enumerate((10.0, 30.0, 50.0)):
for metric, value in (("cpu_pct", cpu), ("mem_pct", 20.0),
("mem_usage_bytes", 1000.0)):
for metric, value in (("cpu_pct", cpu), ("mem_used_pct", 20.0),
("mem_used_bytes", 1000.0)):
await s.execute(text(
"INSERT INTO plugin_metrics (id, source_module, resource_name, "
"metric_name, value, recorded_at) "
@@ -201,6 +201,10 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
c = await dtools.get_container(web, range="1h")
assert c["host_name"] == host_name and c["status"] == "running"
assert c["cpu_pct_history"]["max"] == 50.0 and c["cpu_pct_history"]["points"] == 3
assert c["mem_used_pct_history"]["max"] == 20.0
# File cache is in the one container's detail, not in the list rows (#6030).
assert c["mem_used_bytes"] == 1000 and c["mem_cache_bytes"] == 5000
assert "mem_cache_bytes" not in proj[0]
crashed = await dtools.get_container(job, host=host_name)
assert [e["event"] for e in crashed["events"]] == ["die"]
# A unique part of a long swarm task name is enough.