fix(docker): a container's memory is its processes' own; file cache shown on its page (#6030)

The agent sent Docker's working set (usage less inactive_file), which still
counts recently used file cache. A file-heavy container with no limit read
191 GB while its processes held 2.2 GB, and the host's own figure, which
already leaves cache out, disagreed with it.

Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so
Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes
apart. The new names keep the two meanings apart in history and in learned
normals: docker_013 renames the columns, clears the old values and adds
mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers,
container detail, normals and the MCP tools all read the new figure. File
cache is shown only on the container's page and in get_container.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -44,10 +44,11 @@ def setup(app: "Quart") -> None:
         required_role=UserRole.viewer,
     )
     # Each container's CPU (100% is one core) and memory share learn their
-    # normal (#5742), each with its floor from Settings → Thresholds.
+    # normal (#5742), each with its floor from Settings → Thresholds. Memory
+    # is its processes' own, file cache left out (#6030).
     from steward.core.norms import floor_setting, learn_normal
     learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct"))
-    learn_normal("docker", "mem_pct", floor_setting(app, "container_mem_pct"))
+    learn_normal("docker", "mem_used_pct", floor_setting(app, "container_mem_pct"))
     # A host's agent carries its containers' figures: they go silent with it (#5721).
     from steward.core.alerts import agent_fed
     agent_fed("docker", lambda resource: resource.split("/", 1)[0])
@@ -94,4 +95,4 @@ def get_alert_metrics() -> dict[str, list[str]]:
     aside (#5788, swarm_health.py)."""
     from .disk_view import DISK_METRICS
     from .swarm_health import SWARM_METRICS
-    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]}
+    return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]}

plugins/docker/ingest.py

@@ -345,8 +345,8 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
     one entry per ingested sample carrying docker data (usually one; more when
     the agent flushes a backlog). Every snapshot's running containers record
-    cpu_pct, mem_pct and mem_usage_bytes into the one metric store, at the
-    snapshot's time; the newest snapshot drives current container state and
+    cpu_pct, mem_used_pct, mem_used_bytes and mem_cache_bytes into the one
+    metric store, at the snapshot's time; the newest snapshot drives current container state and
     lifecycle-event derivation. `swarm` is the newest
     sample's swarm object (or None off managers) — persisted when present.
     `disk` is the newest sample's /system/df summary (or None on Docker-less
@@ -400,7 +400,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
             if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
                 continue
             resource = f"{host.name}/{c['name']}"
-            for metric in ("cpu_pct", "mem_pct", "mem_usage_bytes"):
+            for metric in ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes"):
                 if c.get(metric) is not None:
                     batch.add(source_module="docker", resource_name=resource,
                               metric_name=metric, value=c[metric], recorded_at=recorded_at)
@@ -444,9 +444,10 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         existing.image = c.get("image", "") or ""
         existing.status = c.get("status", "unknown") or "unknown"
         existing.cpu_pct = c.get("cpu_pct")
-        existing.mem_usage_bytes = c.get("mem_usage_bytes")
+        existing.mem_used_bytes = c.get("mem_used_bytes")
         existing.mem_limit_bytes = c.get("mem_limit_bytes")
-        existing.mem_pct = c.get("mem_pct")
+        existing.mem_used_pct = c.get("mem_used_pct")
+        existing.mem_cache_bytes = c.get("mem_cache_bytes")
         existing.ports_json = json.dumps(c.get("ports") or [])
         existing.started_at = _parse_started_at(c.get("started_at"))
         existing.scraped_at = latest_at

plugins/docker/mcp_tools.py

@@ -62,8 +62,8 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime,
         "oom_killed": bool(c.oom_killed),
         "restart_count": c.restart_count,
         "cpu_pct": rnd(c.cpu_pct),
-        "mem_pct": rnd(c.mem_pct),
-        "mem_usage_bytes": c.mem_usage_bytes,
+        "mem_used_pct": rnd(c.mem_used_pct),
+        "mem_used_bytes": c.mem_used_bytes,
         "mem_limit_bytes": c.mem_limit_bytes,
         "started_at": iso(c.started_at),
         "uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
@@ -388,9 +388,10 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
       a name also works (swarm task names are long).
     host: the host's name or id. Needed only when several hosts run a
       container with this name.
-    range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
-      max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
-      "7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
+    range: the window for `cpu_pct_history` and `mem_used_pct_history` (min,
+      avg, max, p95, last and a ≤60-point series) — "1h", "6h", "24h"
+      (default), "7d" or "30d". `cpu_pct` and `mem_used_pct` are the latest
+      reported values.
     events: how many lifecycle events (start, restart, stop, die, oom, health_change)
       to return, newest first (default 20, max 100).
 
@@ -399,6 +400,12 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
     removed container, while Steward retains its logs or events: the reply
     has `removed: true`, `last_seen_at`, its history and its events, and no
     live state. A live container has `removed: false`.
+
+    Memory (`mem_used_bytes`, `mem_used_pct`, of its limit or of the host's
+    RAM when it has none) is what its processes hold: anon + shmem.
+    `mem_cache_bytes` is the file cache charged to it, which the kernel
+    gives back when the host needs the memory, so a large figure there is
+    not memory the container needs.
     """
     since, key = lookback(range)
     events = clamp_limit(events, 20, 100)
@@ -412,7 +419,7 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
             db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
         host_names = await _host_names(db)
         cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
-        mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
+        mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since)
         evs = await docker_svc.container_events(db, c.host_id, c.name, events)
         sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
     hour_ago = now - timedelta(hours=1)
@@ -430,9 +437,10 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
         "ports": ports,
         "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
         "blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
+        "mem_cache_bytes": c.mem_cache_bytes,
         "range": key,
         "cpu_pct_history": summarize(cpu),
-        "mem_pct_history": summarize(mem),
+        "mem_used_pct_history": summarize(mem),
         "events": _event_rows(evs, now),
         "next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
     }
@@ -448,7 +456,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i
     """get_container for a container Docker has removed: what Steward kept."""
     host_names = await _host_names(db)
     cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
-    mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
+    mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since)
     evs = await docker_svc.container_events(db, c.host_id, c.name, events)
     return {
         "container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
@@ -456,7 +464,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i
         "last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
         "range": key,
         "cpu_pct_history": summarize(cpu),
-        "mem_pct_history": summarize(mem),
+        "mem_used_pct_history": summarize(mem),
         "events": _event_rows(evs, now),
         "next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
     }
@@ -536,7 +544,7 @@ async def list_swarm_services(limit: int = 100) -> dict:
                 {"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
                 {"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
                  "status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
-                 "cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
+                 "cpu_pct": rnd(r["cpu_pct"]), "mem_used_pct": rnd(r["mem_pct"])}
                 for r in s["replicas"]
             ],
         }

plugins/docker/migrations/versions/docker_013_mem_used.py

@@ -0,0 +1,36 @@
+"""A container's memory is its processes' own, with file cache beside it
+
+Renames docker_containers.mem_usage_bytes / mem_pct to mem_used_bytes /
+mem_used_pct and adds mem_cache_bytes (#6030). The old figure was Docker's
+working set (usage less idle file cache), which still counted recently used
+file cache: a container that reads a lot of files showed hundreds of GB it
+didn't need. The new figure is anon + shmem, what the processes hold and
+can't hand back without swap, read by agent 1.23.0. The old values are
+cleared rather than carried under the new names; each container's next
+report from an updated agent fills them.
+
+Revision ID: docker_013_mem_used
+Revises: docker_012_drop_metrics
+Create Date: 2026-10-18
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_013_mem_used"
+down_revision: Union[str, None] = "docker_012_drop_metrics"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column("docker_containers", "mem_usage_bytes", new_column_name="mem_used_bytes")
+    op.alter_column("docker_containers", "mem_pct", new_column_name="mem_used_pct")
+    op.add_column("docker_containers", sa.Column("mem_cache_bytes", sa.BigInteger, nullable=True))
+    op.execute("UPDATE docker_containers SET mem_used_bytes = NULL, mem_used_pct = NULL")
+
+
+def downgrade() -> None:
+    op.drop_column("docker_containers", "mem_cache_bytes")
+    op.alter_column("docker_containers", "mem_used_pct", new_column_name="mem_pct")
+    op.alter_column("docker_containers", "mem_used_bytes", new_column_name="mem_usage_bytes")

plugins/docker/models.py

@@ -29,10 +29,14 @@ class DockerContainer(Base):
     # running | stopped | paused | exited | dead
     cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
     # BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes
-    # (~2.1 GB), which overflows int4 and fails the insert.
-    mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    # (~2.1 GB), which overflows int4 and fails the insert. Used is what its
+    # processes hold (anon + shmem); file cache is kept apart, since the kernel
+    # gives it back when the host needs it (#6030). used_pct is of the limit,
+    # which is the host's RAM when none is set.
+    mem_used_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
     mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
-    mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
+    mem_used_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
+    mem_cache_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
     restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
     # JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]

plugins/docker/service.py

@@ -161,9 +161,10 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No
 
     Docker's CPU% is per core (100% = one core busy), so it is shown as cores
     and graded as a share of the host's CPUs when the host's core count is
-    known. Docker's memory limit is the host's RAM when none is set, so memory
-    is shown in bytes, against a limit only when one is actually set; mem_pct
-    is then a share of that limit, otherwise of the host's memory. `cpu_pct`
+    known. Memory is what the container's processes hold, file cache left out
+    (#6030). Docker's memory limit is the host's RAM when none is set, so
+    memory is shown in bytes, against a limit only when one is actually set;
+    mem_pct is then a share of that limit, otherwise of the host's memory. `cpu_pct`
     replaces the container's last sample (an average, say) when given.
 
     With `thresholds`, cpu_grade and mem_grade are the grades to show: past
@@ -178,9 +179,9 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No
         "cores": None if cpu is None else cpu / 100.0,
         "cpu_share": None if cpu is None or not cpus else cpu / cpus,
         "cpus": cpus,
-        "mem_bytes": getattr(c, "mem_usage_bytes", None),
+        "mem_bytes": getattr(c, "mem_used_bytes", None),
         "mem_limit": limit if limited else None,
-        "mem_pct": getattr(c, "mem_pct", None),
+        "mem_pct": getattr(c, "mem_used_pct", None),
         "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
         "cpu_grade": None, "cpu_usual": None, "mem_grade": None, "mem_usual": None,
     }
@@ -191,7 +192,7 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No
 
 # Container figures graded against their own normal (#5742): the metric that
 # learns it and its floor's setting (norms.floor.*).
-NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_pct", "container_mem_pct")}
+NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_used_pct", "container_mem_pct")}
 
 
 def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None:
@@ -209,7 +210,7 @@ def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None:
     u["cpu_grade"] = grade
     if u["mem_of"]:
         u["mem_grade"], u["mem_usual"] = graded(
-            u["mem_pct"], "mem", thresholds, norms.get("mem_pct"),
+            u["mem_pct"], "mem", thresholds, norms.get("mem_used_pct"),
             floor_of(thresholds, "container_mem_pct"), "%")
     else:
         u["mem_grade"] = "dim"
@@ -318,7 +319,7 @@ async def query_logs(db, host_id: str, name: str, stream: str, query: str,
     return (await db.execute(stmt)).all()
 
 
-CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
+CONTAINER_SERIES = ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes")
 
 
 # A container's series live in the one metric store (#5752): source "docker",
@@ -397,14 +398,14 @@ def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetim
 
 
 async def recent_usage(db, keys, since: datetime, width: int) -> dict:
-    """{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}}
+    """{(host_id, container_name): {bucket start: (cpu_pct, mem_used_bytes)}}
     averaged per bucket since `since`, for the given containers, in one query."""
     resources = await container_resources(db, keys)
     if not resources:
         return {}
     out: dict = {}
     for resource, at, metric, value in await _samples(
-            db, resources, ("cpu_pct", "mem_usage_bytes"), since, width):
+            db, resources, ("cpu_pct", "mem_used_bytes"), since, width):
         cpu, mem = out.setdefault(resources[resource], {}).get(at, (None, None))
         v = None if value is None else float(value)
         out[resources[resource]][at] = (v, mem) if metric == "cpu_pct" else (cpu, v)
@@ -595,7 +596,7 @@ async def service_history(db, containers, since: datetime) -> list:
         return []
     joined: dict[tuple[str, datetime], list] = {}
     for resource, at, metric, value in await _samples(
-            db, resources, ("cpu_pct", "mem_usage_bytes"), since):
+            db, resources, ("cpu_pct", "mem_used_bytes"), since):
         pair = joined.setdefault((resource, at), [None, None])
         pair[0 if metric == "cpu_pct" else 1] = value
     rows = [(resource, at, cpu, mem) for (resource, at), (cpu, mem) in

plugins/docker/swarm_nodes.py

@@ -36,7 +36,7 @@ def node_cards(nodes, services, containers, *, node_host: dict, host_names: dict
                              "measured": True})
         row["tasks"] += 1
         row["cpu_pct"] += c.cpu_pct or 0.0
-        row["mem_bytes"] += c.mem_usage_bytes or 0
+        row["mem_bytes"] += c.mem_used_bytes or 0
     placed: dict[str, dict[str, int]] = {}
     for s in services:
         for p in _placement(s):

plugins/docker/swarm_view.py

@@ -289,8 +289,8 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
                 "name": c.name,
                 "status": getattr(c, "status", "unknown"),
                 "cpu_pct": getattr(c, "cpu_pct", None),
-                "mem_pct": getattr(c, "mem_pct", None),
-                "mem_bytes": getattr(c, "mem_usage_bytes", None),
+                "mem_pct": getattr(c, "mem_used_pct", None),
+                "mem_bytes": getattr(c, "mem_used_bytes", None),
                 "started_at": getattr(c, "started_at", None),
                 "image": getattr(c, "image", None) or None,
                 "health": getattr(c, "health", None),

plugins/docker/templates/docker/_usage.html

@@ -2,7 +2,8 @@
 
     Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
     toned by its share of the host's CPUs when the host's core count is known.
-    Memory reads in bytes, against a limit only when one is actually set;
+    Memory is what its processes hold, file cache left out (#6030), and
+    reads in bytes, against a limit only when one is actually set;
     mem_pct is then a share of that limit, otherwise of the host's memory.
     Each is toned by usage()'s grade when it has one: within the container's
     own normal for the hour reads healthy, and says what is usual (#5742). -#}

plugins/docker/templates/docker/container_detail.html

@@ -78,6 +78,10 @@
       {%- if c.exit_code is not none and c.status == 'running' and c.restart_count %} <span class="tone-dim">· last exit {{ c.exit_code }}</span>{% endif %}</dd>
     {% endif %}
     <dt>Memory limit</dt><dd>{% if usage.mem_limit %}{{ usage.mem_limit|bytes }}{% else %}<span class="tone-dim">none set</span>{% endif %}</dd>
+    {#- Its memory figure is what its processes hold; the file cache charged to
+        it is shown here only, since the kernel gives it back when the host
+        needs it (#6030). #}
+    <dt>File cache</dt><dd>{% if c.mem_cache_bytes is not none %}{{ c.mem_cache_bytes|bytes }} <span class="tone-dim">files it read or wrote, kept in memory until the host needs it back; not counted in its memory</span>{% else %}<span class="tone-dim">—</span>{% endif %}</dd>
     {% if ports %}
     <dt>Ports</dt><dd class="mono">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</dd>
     {% endif %}