From 56063f27a48373859de696b88a475fca9f0216e6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 17 Oct 2026 21:27:29 -0400 Subject: [PATCH] fix(docker): a container's memory is its processes' own; file cache shown on its page (#6030) The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- docs/plugins/host-agent-design.md | 6 +++ plugins/docker/__init__.py | 7 +-- plugins/docker/ingest.py | 11 +++-- plugins/docker/mcp_tools.py | 28 +++++++---- .../versions/docker_013_mem_used.py | 36 ++++++++++++++ plugins/docker/models.py | 10 ++-- plugins/docker/service.py | 23 ++++----- plugins/docker/swarm_nodes.py | 2 +- plugins/docker/swarm_view.py | 4 +- plugins/docker/templates/docker/_usage.html | 3 +- .../templates/docker/container_detail.html | 4 ++ plugins/host_agent/agent.py | 40 ++++++++++------ steward/core/formatting.py | 6 +-- tests/core/test_norm_grading.py | 6 +-- tests/integration/test_docker.py | 38 +++++++++------ tests/integration/test_docker_recent_usage.py | 2 +- tests/integration/test_docker_service_page.py | 4 +- .../test_host_agent_docker_status.py | 2 +- tests/integration/test_mcp_plugin_tools.py | 16 ++++--- tests/plugins/docker/test_docker_page.py | 2 +- tests/plugins/docker/test_mcp_tools.py | 4 +- tests/plugins/docker/test_overview.py | 16 ++++++- tests/plugins/docker/test_service.py | 4 +- tests/plugins/docker/test_swarm_view.py | 2 +- tests/plugins/host_agent/test_agent_docker.py | 47 +++++++++++++++---- 26 files changed, 226 insertions(+), 99 deletions(-) create mode 100644 plugins/docker/migrations/versions/docker_013_mem_used.py diff --git a/docs/architecture.md b/docs/architecture.md index 85e10cb..3bfeb32 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -119,7 +119,7 @@ What learns a normal, and the floor each departure must clear (Settings → Thre | Host memory | `host_agent` / `mem_used_pct` | 5 points | | Host load per core | `host_agent` / `load_pct` (load_1m ÷ cores × 100, recorded at ingest) | 10 points | | Container CPU | `docker` / `cpu_pct` (100% is one core) | 10 points of one core | -| Container memory | `docker` / `mem_pct` (of its limit, or of host RAM when it has none) | 5 points | +| Container memory | `docker` / `mem_used_pct` (its processes' own memory, file cache left out, of its limit or of host RAM when it has none) | 5 points | | Monitor response time | `` / `response_ms` | 100 ms | | Router 4xx share, p95 | `traefik` / `error_rate_4xx_pct`, `latency_p95_ms` | 2 points, 50 ms | diff --git a/docs/plugins/host-agent-design.md b/docs/plugins/host-agent-design.md index 75cc8ca..b1499b0 100644 --- a/docs/plugins/host-agent-design.md +++ b/docs/plugins/host-agent-design.md @@ -130,6 +130,12 @@ Each mount is read in a daemon thread under one 5 s deadline. A hard-mounted NFS Steward stores them per mount (`nfs_*` on `:`), and the slowest mount's `exe_ms` on the host as `nfs_exe_max_ms`, for a rule ("NFS slow": over 1000 ms for 3 checks) and for the host page's NFS chart beside `mounts_stalled`. Each mount's bar on the host page says what a request took and how many were sent again or timed out. A request is counted when it finishes, so a stuck share shows nothing until it comes back, then its long requests all at once. +### A container's memory is its processes' own (1.23.0, #6030) + +Each running container sends `mem_used_bytes` and `mem_used_pct` (of its limit, or of the host's RAM when it has none) and, beside them, `mem_cache_bytes`. Used is anon plus shmem from the container's cgroup memory stats (v1: rss plus shmem). It is what the processes hold and can't give back without swap, and it counts Postgres's shared buffers, which live in shmem. Cache is the file cache charged to the container (v2's `file`, v1's `cache`, shmem taken out). The kernel keeps what the container read or wrote for as long as nothing else needs the memory, and gives it back when something does. + +Before this, the agent sent Docker's working set (usage less `inactive_file`) as `mem_usage_bytes`. That figure still counts recently used file cache. On 2026-10-17 a file-heavy container with no memory limit read 191 GB that way, while its processes held 2.2 GB. The host's own figure (`MemTotal − MemAvailable`) already left cache out, so the two disagreed. The new names keep the two meanings apart in history and in learned normals. + ### Every Docker read is bounded (1.15.0, #5764) One Docker answer must arrive within 15 s and fit in 32 MiB, and one log answer within 5 s and 4 MiB. Past either limit the read is cut off, not waited on. A per-read timeout alone never ends a body that keeps trickling in. Container logs are read since each container's cursor, but only its newest 2,000 lines, all containers within 10 s. A container that writes more than that gets a `_steward` note saying older lines were skipped. A container whose answer is too big skips ahead to now. Before this, a 1.13.0 agent re-read a growing backlog every interval and went silent for a day at 2.1 GB. Each sample also carries the agent's own `agent` footprint (`rss_bytes`, `cpu_pct`), which Steward records as `agent_rss_bytes` and `agent_cpu_pct`. diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index 7183969..d667eda 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -44,10 +44,11 @@ def setup(app: "Quart") -> None: required_role=UserRole.viewer, ) # Each container's CPU (100% is one core) and memory share learn their - # normal (#5742), each with its floor from Settings → Thresholds. + # normal (#5742), each with its floor from Settings → Thresholds. Memory + # is its processes' own, file cache left out (#6030). from steward.core.norms import floor_setting, learn_normal learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct")) - learn_normal("docker", "mem_pct", floor_setting(app, "container_mem_pct")) + learn_normal("docker", "mem_used_pct", floor_setting(app, "container_mem_pct")) # A host's agent carries its containers' figures: they go silent with it (#5721). from steward.core.alerts import agent_fed agent_fed("docker", lambda resource: resource.split("/", 1)[0]) @@ -94,4 +95,4 @@ def get_alert_metrics() -> dict[str, list[str]]: aside (#5788, swarm_health.py).""" from .disk_view import DISK_METRICS from .swarm_health import SWARM_METRICS - return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]} + return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]} diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py index 027ce95..903606f 100644 --- a/plugins/docker/ingest.py +++ b/plugins/docker/ingest.py @@ -345,8 +345,8 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None, `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) — one entry per ingested sample carrying docker data (usually one; more when the agent flushes a backlog). Every snapshot's running containers record - cpu_pct, mem_pct and mem_usage_bytes into the one metric store, at the - snapshot's time; the newest snapshot drives current container state and + cpu_pct, mem_used_pct, mem_used_bytes and mem_cache_bytes into the one + metric store, at the snapshot's time; the newest snapshot drives current container state and lifecycle-event derivation. `swarm` is the newest sample's swarm object (or None off managers) — persisted when present. `disk` is the newest sample's /system/df summary (or None on Docker-less @@ -400,7 +400,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None, if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"): continue resource = f"{host.name}/{c['name']}" - for metric in ("cpu_pct", "mem_pct", "mem_usage_bytes"): + for metric in ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes"): if c.get(metric) is not None: batch.add(source_module="docker", resource_name=resource, metric_name=metric, value=c[metric], recorded_at=recorded_at) @@ -444,9 +444,10 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None, existing.image = c.get("image", "") or "" existing.status = c.get("status", "unknown") or "unknown" existing.cpu_pct = c.get("cpu_pct") - existing.mem_usage_bytes = c.get("mem_usage_bytes") + existing.mem_used_bytes = c.get("mem_used_bytes") existing.mem_limit_bytes = c.get("mem_limit_bytes") - existing.mem_pct = c.get("mem_pct") + existing.mem_used_pct = c.get("mem_used_pct") + existing.mem_cache_bytes = c.get("mem_cache_bytes") existing.ports_json = json.dumps(c.get("ports") or []) existing.started_at = _parse_started_at(c.get("started_at")) existing.scraped_at = latest_at diff --git a/plugins/docker/mcp_tools.py b/plugins/docker/mcp_tools.py index 5a3b5f5..15adc85 100644 --- a/plugins/docker/mcp_tools.py +++ b/plugins/docker/mcp_tools.py @@ -62,8 +62,8 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime, "oom_killed": bool(c.oom_killed), "restart_count": c.restart_count, "cpu_pct": rnd(c.cpu_pct), - "mem_pct": rnd(c.mem_pct), - "mem_usage_bytes": c.mem_usage_bytes, + "mem_used_pct": rnd(c.mem_used_pct), + "mem_used_bytes": c.mem_used_bytes, "mem_limit_bytes": c.mem_limit_bytes, "started_at": iso(c.started_at), "uptime_s": age_s(c.started_at, now) if running and c.started_at else None, @@ -388,9 +388,10 @@ async def get_container(container: str, host: str = "", range: str = "24h", even a name also works (swarm task names are long). host: the host's name or id. Needed only when several hosts run a container with this name. - range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg, - max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default), - "7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values. + range: the window for `cpu_pct_history` and `mem_used_pct_history` (min, + avg, max, p95, last and a ≤60-point series) — "1h", "6h", "24h" + (default), "7d" or "30d". `cpu_pct` and `mem_used_pct` are the latest + reported values. events: how many lifecycle events (start, restart, stop, die, oom, health_change) to return, newest first (default 20, max 100). @@ -399,6 +400,12 @@ async def get_container(container: str, host: str = "", range: str = "24h", even removed container, while Steward retains its logs or events: the reply has `removed: true`, `last_seen_at`, its history and its events, and no live state. A live container has `removed: false`. + + Memory (`mem_used_bytes`, `mem_used_pct`, of its limit or of the host's + RAM when it has none) is what its processes hold: anon + shmem. + `mem_cache_bytes` is the file cache charged to it, which the kernel + gives back when the host needs the memory, so a large figure there is + not memory the container needs. """ since, key = lookback(range) events = clamp_limit(events, 20, 100) @@ -412,7 +419,7 @@ async def get_container(container: str, host: str = "", range: str = "24h", even db, now - timedelta(hours=24))).get((c.host_id, c.name), []) host_names = await _host_names(db) cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since) - mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since) + mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since) evs = await docker_svc.container_events(db, c.host_id, c.name, events) sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)]) hour_ago = now - timedelta(hours=1) @@ -430,9 +437,10 @@ async def get_container(container: str, host: str = "", range: str = "24h", even "ports": ports, "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes, "blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes, + "mem_cache_bytes": c.mem_cache_bytes, "range": key, "cpu_pct_history": summarize(cpu), - "mem_pct_history": summarize(mem), + "mem_used_pct_history": summarize(mem), "events": _event_rows(evs, now), "next": f'search_container_logs(container="{c.name}", host="{c.host_id}")', } @@ -448,7 +456,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i """get_container for a container Docker has removed: what Steward kept.""" host_names = await _host_names(db) cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since) - mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since) + mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since) evs = await docker_svc.container_events(db, c.host_id, c.name, events) return { "container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id), @@ -456,7 +464,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i "last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now), "range": key, "cpu_pct_history": summarize(cpu), - "mem_pct_history": summarize(mem), + "mem_used_pct_history": summarize(mem), "events": _event_rows(evs, now), "next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")', } @@ -536,7 +544,7 @@ async def list_swarm_services(limit: int = 100) -> dict: {"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else {"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"], "status": r["status"], "health": r["health"], "restart_count": r["restart_count"], - "cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])} + "cpu_pct": rnd(r["cpu_pct"]), "mem_used_pct": rnd(r["mem_pct"])} for r in s["replicas"] ], } diff --git a/plugins/docker/migrations/versions/docker_013_mem_used.py b/plugins/docker/migrations/versions/docker_013_mem_used.py new file mode 100644 index 0000000..7978049 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_013_mem_used.py @@ -0,0 +1,36 @@ +"""A container's memory is its processes' own, with file cache beside it + +Renames docker_containers.mem_usage_bytes / mem_pct to mem_used_bytes / +mem_used_pct and adds mem_cache_bytes (#6030). The old figure was Docker's +working set (usage less idle file cache), which still counted recently used +file cache: a container that reads a lot of files showed hundreds of GB it +didn't need. The new figure is anon + shmem, what the processes hold and +can't hand back without swap, read by agent 1.23.0. The old values are +cleared rather than carried under the new names; each container's next +report from an updated agent fills them. + +Revision ID: docker_013_mem_used +Revises: docker_012_drop_metrics +Create Date: 2026-10-18 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_013_mem_used" +down_revision: Union[str, None] = "docker_012_drop_metrics" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("docker_containers", "mem_usage_bytes", new_column_name="mem_used_bytes") + op.alter_column("docker_containers", "mem_pct", new_column_name="mem_used_pct") + op.add_column("docker_containers", sa.Column("mem_cache_bytes", sa.BigInteger, nullable=True)) + op.execute("UPDATE docker_containers SET mem_used_bytes = NULL, mem_used_pct = NULL") + + +def downgrade() -> None: + op.drop_column("docker_containers", "mem_cache_bytes") + op.alter_column("docker_containers", "mem_used_pct", new_column_name="mem_pct") + op.alter_column("docker_containers", "mem_used_bytes", new_column_name="mem_usage_bytes") diff --git a/plugins/docker/models.py b/plugins/docker/models.py index 2b63aa5..22517a6 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -29,10 +29,14 @@ class DockerContainer(Base): # running | stopped | paused | exited | dead cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True) # BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes - # (~2.1 GB), which overflows int4 and fails the insert. - mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + # (~2.1 GB), which overflows int4 and fails the insert. Used is what its + # processes hold (anon + shmem); file cache is kept apart, since the kernel + # gives it back when the host needs it (#6030). used_pct is of the limit, + # which is the host's RAM when none is set. + mem_used_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) - mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + mem_used_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + mem_cache_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") # JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] diff --git a/plugins/docker/service.py b/plugins/docker/service.py index 050f5f5..accb7a1 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -161,9 +161,10 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No Docker's CPU% is per core (100% = one core busy), so it is shown as cores and graded as a share of the host's CPUs when the host's core count is - known. Docker's memory limit is the host's RAM when none is set, so memory - is shown in bytes, against a limit only when one is actually set; mem_pct - is then a share of that limit, otherwise of the host's memory. `cpu_pct` + known. Memory is what the container's processes hold, file cache left out + (#6030). Docker's memory limit is the host's RAM when none is set, so + memory is shown in bytes, against a limit only when one is actually set; + mem_pct is then a share of that limit, otherwise of the host's memory. `cpu_pct` replaces the container's last sample (an average, say) when given. With `thresholds`, cpu_grade and mem_grade are the grades to show: past @@ -178,9 +179,9 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No "cores": None if cpu is None else cpu / 100.0, "cpu_share": None if cpu is None or not cpus else cpu / cpus, "cpus": cpus, - "mem_bytes": getattr(c, "mem_usage_bytes", None), + "mem_bytes": getattr(c, "mem_used_bytes", None), "mem_limit": limit if limited else None, - "mem_pct": getattr(c, "mem_pct", None), + "mem_pct": getattr(c, "mem_used_pct", None), "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None), "cpu_grade": None, "cpu_usual": None, "mem_grade": None, "mem_usual": None, } @@ -191,7 +192,7 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No # Container figures graded against their own normal (#5742): the metric that # learns it and its floor's setting (norms.floor.*). -NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_pct", "container_mem_pct")} +NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_used_pct", "container_mem_pct")} def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None: @@ -209,7 +210,7 @@ def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None: u["cpu_grade"] = grade if u["mem_of"]: u["mem_grade"], u["mem_usual"] = graded( - u["mem_pct"], "mem", thresholds, norms.get("mem_pct"), + u["mem_pct"], "mem", thresholds, norms.get("mem_used_pct"), floor_of(thresholds, "container_mem_pct"), "%") else: u["mem_grade"] = "dim" @@ -318,7 +319,7 @@ async def query_logs(db, host_id: str, name: str, stream: str, query: str, return (await db.execute(stmt)).all() -CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes") +CONTAINER_SERIES = ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes") # A container's series live in the one metric store (#5752): source "docker", @@ -397,14 +398,14 @@ def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetim async def recent_usage(db, keys, since: datetime, width: int) -> dict: - """{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}} + """{(host_id, container_name): {bucket start: (cpu_pct, mem_used_bytes)}} averaged per bucket since `since`, for the given containers, in one query.""" resources = await container_resources(db, keys) if not resources: return {} out: dict = {} for resource, at, metric, value in await _samples( - db, resources, ("cpu_pct", "mem_usage_bytes"), since, width): + db, resources, ("cpu_pct", "mem_used_bytes"), since, width): cpu, mem = out.setdefault(resources[resource], {}).get(at, (None, None)) v = None if value is None else float(value) out[resources[resource]][at] = (v, mem) if metric == "cpu_pct" else (cpu, v) @@ -595,7 +596,7 @@ async def service_history(db, containers, since: datetime) -> list: return [] joined: dict[tuple[str, datetime], list] = {} for resource, at, metric, value in await _samples( - db, resources, ("cpu_pct", "mem_usage_bytes"), since): + db, resources, ("cpu_pct", "mem_used_bytes"), since): pair = joined.setdefault((resource, at), [None, None]) pair[0 if metric == "cpu_pct" else 1] = value rows = [(resource, at, cpu, mem) for (resource, at), (cpu, mem) in diff --git a/plugins/docker/swarm_nodes.py b/plugins/docker/swarm_nodes.py index c8f7a75..a7644c0 100644 --- a/plugins/docker/swarm_nodes.py +++ b/plugins/docker/swarm_nodes.py @@ -36,7 +36,7 @@ def node_cards(nodes, services, containers, *, node_host: dict, host_names: dict "measured": True}) row["tasks"] += 1 row["cpu_pct"] += c.cpu_pct or 0.0 - row["mem_bytes"] += c.mem_usage_bytes or 0 + row["mem_bytes"] += c.mem_used_bytes or 0 placed: dict[str, dict[str, int]] = {} for s in services: for p in _placement(s): diff --git a/plugins/docker/swarm_view.py b/plugins/docker/swarm_view.py index 984852a..8b1d4fa 100644 --- a/plugins/docker/swarm_view.py +++ b/plugins/docker/swarm_view.py @@ -289,8 +289,8 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d "name": c.name, "status": getattr(c, "status", "unknown"), "cpu_pct": getattr(c, "cpu_pct", None), - "mem_pct": getattr(c, "mem_pct", None), - "mem_bytes": getattr(c, "mem_usage_bytes", None), + "mem_pct": getattr(c, "mem_used_pct", None), + "mem_bytes": getattr(c, "mem_used_bytes", None), "started_at": getattr(c, "started_at", None), "image": getattr(c, "image", None) or None, "health": getattr(c, "health", None), diff --git a/plugins/docker/templates/docker/_usage.html b/plugins/docker/templates/docker/_usage.html index 8d0a62a..a69fdaf 100644 --- a/plugins/docker/templates/docker/_usage.html +++ b/plugins/docker/templates/docker/_usage.html @@ -2,7 +2,8 @@ Docker counts CPU per core (100% = one core busy), so CPU reads as cores, toned by its share of the host's CPUs when the host's core count is known. - Memory reads in bytes, against a limit only when one is actually set; + Memory is what its processes hold, file cache left out (#6030), and + reads in bytes, against a limit only when one is actually set; mem_pct is then a share of that limit, otherwise of the host's memory. Each is toned by usage()'s grade when it has one: within the container's own normal for the hour reads healthy, and says what is usual (#5742). -#} diff --git a/plugins/docker/templates/docker/container_detail.html b/plugins/docker/templates/docker/container_detail.html index 1cd08f7..fa399f8 100644 --- a/plugins/docker/templates/docker/container_detail.html +++ b/plugins/docker/templates/docker/container_detail.html @@ -78,6 +78,10 @@ {%- if c.exit_code is not none and c.status == 'running' and c.restart_count %} · last exit {{ c.exit_code }}{% endif %} {% endif %}
Memory limit
{% if usage.mem_limit %}{{ usage.mem_limit|bytes }}{% else %}none set{% endif %}
+ {#- Its memory figure is what its processes hold; the file cache charged to + it is shown here only, since the kernel gives it back when the host + needs it (#6030). #} +
File cache
{% if c.mem_cache_bytes is not none %}{{ c.mem_cache_bytes|bytes }} files it read or wrote, kept in memory until the host needs it back; not counted in its memory{% else %}—{% endif %}
{% if ports %}
Ports
{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}
{% endif %} diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 43706ea..8f9f3af 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -22,7 +22,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor, wait as wait_futures from datetime import datetime, timezone -AGENT_VERSION = "1.22.0" +AGENT_VERSION = "1.23.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -721,19 +721,30 @@ def _docker_cpu_pct(stats: dict) -> float: def _docker_mem(stats: dict): - """Return (usage_bytes, limit_bytes, mem_pct); working set excludes cache.""" + """Return (used_bytes, limit_bytes, used_pct, cache_bytes) (#6030). + + Used is what the processes hold and can't hand back without swap: anon + plus shmem (Postgres keeps its shared buffers in shmem). File cache is + reported apart: the kernel keeps what a container read or wrote while + nothing else wants the memory, so a file-heavy container's cache can + run to hundreds of GB it doesn't need. Docker's own working set (usage + less inactive_file) still counts the recently used part of it.""" try: mem = stats.get("memory_stats", {}) - usage = mem.get("usage", 0) limit = mem.get("limit", 0) - # Subtract page cache for working set (cgroup v1 "cache", v2 "inactive_file"). - cache = mem.get("stats", {}).get("cache", 0) or \ - mem.get("stats", {}).get("inactive_file", 0) - actual = max(0, usage - cache) - pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0 - return actual, limit, pct + st = mem.get("stats", {}) + if "anon" in st: # cgroup v2 + shmem = st.get("shmem", 0) + used = st["anon"] + shmem + cache = max(0, st.get("file", 0) - shmem) + else: # cgroup v1: rss leaves shmem out, cache has it in + shmem = st.get("total_shmem", st.get("shmem", 0)) + used = st.get("total_rss", st.get("rss", 0)) + shmem + cache = max(0, st.get("total_cache", st.get("cache", 0)) - shmem) + pct = round(used / limit * 100.0, 2) if limit > 0 else 0.0 + return used, limit, pct, cache except (KeyError, TypeError): - return 0, 0, 0.0 + return 0, 0, 0.0, 0 def _docker_ports(ports: list) -> list: @@ -832,13 +843,13 @@ def _collect_one_container(socket_path: str, c: dict) -> dict: except (OSError, ValueError): pass - cpu_pct = mem_usage = mem_limit = mem_pct = None + cpu_pct = mem_used = mem_limit = mem_pct = mem_cache = None net_rx = net_tx = blk_read = blk_write = None if state == "running": try: stats = _docker_request(socket_path, f"/containers/{cid}/stats?stream=false") cpu_pct = _docker_cpu_pct(stats) - mem_usage, mem_limit, mem_pct = _docker_mem(stats) + mem_used, mem_limit, mem_pct, mem_cache = _docker_mem(stats) net_rx, net_tx, blk_read, blk_write = _docker_net_io(stats) except (OSError, ValueError): pass @@ -853,9 +864,10 @@ def _collect_one_container(socket_path: str, c: dict) -> dict: "image": c.get("Image", ""), "status": state, "cpu_pct": cpu_pct, - "mem_usage_bytes": mem_usage, + "mem_used_bytes": mem_used, "mem_limit_bytes": mem_limit, - "mem_pct": mem_pct, + "mem_used_pct": mem_pct, + "mem_cache_bytes": mem_cache, "ports": _docker_ports(c.get("Ports", [])), "started_at": started_at, "finished_at": finished_at, diff --git a/steward/core/formatting.py b/steward/core/formatting.py index d94ffec..ab23dc3 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -318,8 +318,8 @@ METRICS: dict[str, tuple[str, str]] = { "latency_ms": ("WAN latency", "round trip from the gateway"), "total_clients": ("Connected clients", "devices on the network"), "cpu_pct": ("CPU", "CPU in use; for a container, 100% is one core"), - "mem_pct": ("Memory (% of limit)", "memory in use, against the container's limit or the host's RAM"), - "mem_usage_bytes": ("Memory used", "a container's memory in use, in bytes"), + "mem_used_bytes": ("Memory used", "what a container's processes hold, in bytes; file cache not counted"), + "mem_cache_bytes": ("File cache", "files a container read or wrote, kept in memory until the host needs it back"), "restart_count": ("Restarts", "times the container has restarted, in total"), "disk_total_bytes": ("Docker disk use", "everything Docker keeps on the host (resource: the host)"), "disk_images_bytes": ("Docker images", "space Docker's images take on the host"), @@ -327,7 +327,7 @@ METRICS: dict[str, tuple[str, str]] = { "disk_volumes_bytes": ("Docker volumes", "space Docker's volumes take on the host"), "disk_build_cache_bytes": ("Docker build cache", "left by docker build; safe to clear"), "disk_reclaimable_bytes": ("Reclaimable by Docker", "held by images no container uses"), - "mem_used_pct": ("Memory used", "memory in use"), + "mem_used_pct": ("Memory used", "memory in use, file cache not counted; a container's is of its limit, or of the host's RAM when it has none"), "mem_available_bytes": ("Memory available", "memory free for new work"), "swap_used_bytes": ("Swap used", "memory paged out to disk"), "disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host, by space or inodes"), diff --git a/tests/core/test_norm_grading.py b/tests/core/test_norm_grading.py index 52678ca..c38172f 100644 --- a/tests/core/test_norm_grading.py +++ b/tests/core/test_norm_grading.py @@ -101,7 +101,7 @@ def test_a_monitor_down_stays_down_whatever_its_normal(): def test_a_containers_cpu_is_learned_per_core_and_said_in_cores(): # 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual. - c = NS(cpu_pct=380.0, mem_pct=50.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30) + c = NS(cpu_pct=380.0, mem_used_pct=50.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30) u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)}) assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour" assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit" @@ -109,8 +109,8 @@ def test_a_containers_cpu_is_learned_per_core_and_said_in_cores(): def test_a_containers_memory_within_its_normal_reads_healthy(): - c = NS(cpu_pct=1.0, mem_pct=92.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30) - u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_pct": _norm(90.0)}) + c = NS(cpu_pct=1.0, mem_used_pct=92.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30) + u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_used_pct": _norm(90.0)}) assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour") assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit" diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 909efa5..abb075d 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -136,8 +136,9 @@ def test_persist_scopes_containers_by_host(app): now = datetime.now(timezone.utc) snapshot = [(now, [{ "name": "web", "container_id": "abc123", "image": "nginx", - "status": "running", "cpu_pct": 12.5, "mem_pct": 30.0, - "mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None, + "status": "running", "cpu_pct": 12.5, "mem_used_pct": 30.0, + "mem_used_bytes": 100, "mem_limit_bytes": 200, "mem_cache_bytes": 900, + "ports": [], "started_at": None, # enrichment (agent ≥ 1.4.0) "health": "healthy", "restart_count": 2, "exit_code": 0, "oom_killed": False, "compose_project": "stack", "service_name": "web", @@ -165,7 +166,7 @@ def test_persist_scopes_containers_by_host(app): # at the snapshot's time. metric_hosts = (await s.execute(text( "SELECT COUNT(DISTINCT resource_name) FROM plugin_metrics " - "WHERE source_module = 'docker' AND metric_name = 'mem_usage_bytes' " + "WHERE source_module = 'docker' AND metric_name = 'mem_used_bytes' " "AND recorded_at = :at"), {"at": now})).scalar() resources = {r[0] for r in (await s.execute(text( "SELECT DISTINCT resource_name FROM plugin_metrics " @@ -174,13 +175,22 @@ def test_persist_scopes_containers_by_host(app): enrich = (await s.execute(text( "SELECT health, restart_count, service_name, net_rx_bytes " "FROM docker_containers WHERE name = 'web' LIMIT 1"))).first() - return web_rows, metric_hosts, resources, enrich + # File cache is kept beside the memory figure, not in it (#6030). + memory = (await s.execute(text( + "SELECT mem_used_bytes, mem_used_pct, mem_cache_bytes " + "FROM docker_containers WHERE name = 'web' LIMIT 1"))).first() + cache_series = (await s.execute(text( + "SELECT COUNT(*) FROM plugin_metrics WHERE source_module = 'docker' " + "AND metric_name = 'mem_cache_bytes' AND value = 900"))).scalar() + return web_rows, metric_hosts, resources, enrich, memory, cache_series - web_rows, metric_hosts, resources, enrich = asyncio.run(_go()) + web_rows, metric_hosts, resources, enrich, memory, cache_series = asyncio.run(_go()) assert web_rows == 2 # one 'web' per host, keyed by (host_id, name) assert metric_hosts == 2 # time-series rows scoped per host assert resources == {"alpha/web", "beta/web"} assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips + assert memory == (100, 30.0, 900) + assert cache_series == 2 @_NEEDS_DB @@ -325,7 +335,7 @@ def test_persist_logs_respects_toggle_and_exclude(app): @_NEEDS_DB def test_large_memory_values_persist_as_bigint(app): - """A container using >2^31 bytes of RAM must persist. Regression: mem_usage_bytes + """A container using >2^31 bytes of RAM must persist. Regression: mem_used_bytes / mem_limit_bytes were int4 and overflowed (asyncpg 'value out of int32 range'), failing the whole ingest batch for any host with a >2.1 GB container.""" from sqlalchemy import text @@ -337,8 +347,8 @@ def test_large_memory_values_persist_as_bigint(app): big_limit = 17_179_869_184 # 16 GB snapshot = [(now, [{ "name": "bigmem", "container_id": "deadbeef", "image": "postgres", - "status": "running", "cpu_pct": 1.0, "mem_pct": 50.0, - "mem_usage_bytes": big_usage, "mem_limit_bytes": big_limit, + "status": "running", "cpu_pct": 1.0, "mem_used_pct": 50.0, + "mem_used_bytes": big_usage, "mem_limit_bytes": big_limit, "ports": [], "started_at": None, }])] @@ -351,12 +361,12 @@ def test_large_memory_values_persist_as_bigint(app): await s.flush() await persist(s, h, snapshot) row = (await s.execute(text( - "SELECT mem_usage_bytes, mem_limit_bytes FROM docker_containers " + "SELECT mem_used_bytes, mem_limit_bytes FROM docker_containers " "WHERE name = 'bigmem'"))).first() metric = (await s.execute(text( "SELECT value FROM plugin_metrics WHERE source_module = 'docker' " "AND resource_name = 'bigmemhost/bigmem' " - "AND metric_name = 'mem_usage_bytes'"))).scalar() + "AND metric_name = 'mem_used_bytes'"))).scalar() return row, metric row, metric = asyncio.run(_go()) @@ -372,7 +382,7 @@ def test_lifecycle_events_derived_across_snapshots(app): t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc) t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc) running = {"name": "web", "container_id": "abc", "image": "nginx", - "status": "running", "cpu_pct": 5.0, "mem_pct": 10.0, + "status": "running", "cpu_pct": 5.0, "mem_used_pct": 10.0, "restart_count": 0, "exit_code": None, "oom_killed": False, "health": "healthy"} died = {**running, "status": "exited", "cpu_pct": None, @@ -414,7 +424,7 @@ def test_vanished_container_is_reaped(app): persist = _persist_fn(app) t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc) t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc) - base = {"status": "running", "cpu_pct": 1.0, "mem_pct": 1.0, + base = {"status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0, "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None} keep = {**base, "name": "keep"} gone = {**base, "name": "ci-job-123"} @@ -456,7 +466,7 @@ def test_persist_marks_each_part_for_the_slow_ingest_line(app): persist = _persist_fn(app) at = datetime(2026, 10, 6, 23, 0, 0, tzinfo=timezone.utc) - c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0, + c = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0, "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None} logs = [(at, [{"container": "web", "stream": "stdout", "ts": None, "line": "hello"}])] marks = _Marks() @@ -486,7 +496,7 @@ def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app): persist = _persist_fn(app) t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc) t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc) - web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0, + web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0, "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None} async def _go(): diff --git a/tests/integration/test_docker_recent_usage.py b/tests/integration/test_docker_recent_usage.py index b0c75b9..8587e8f 100644 --- a/tests/integration/test_docker_recent_usage.py +++ b/tests/integration/test_docker_recent_usage.py @@ -47,7 +47,7 @@ def test_recent_usage_averages_each_containers_bucket(app): when = at + timedelta(seconds=secs) return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{name}", metric_name=metric, value=value, recorded_at=when) - for metric, value in (("cpu_pct", cpu), ("mem_usage_bytes", mem))] + for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))] async def go(): async with app.db_sessionmaker() as s: diff --git a/tests/integration/test_docker_service_page.py b/tests/integration/test_docker_service_page.py index 7e15c2c..43d7e49 100644 --- a/tests/integration/test_docker_service_page.py +++ b/tests/integration/test_docker_service_page.py @@ -61,7 +61,7 @@ def test_a_service_page_reads_every_task(app): when = now - timedelta(minutes=mins) return [PluginMetric(source_module="docker", resource_name=f"dock-{tag}/{cname}", metric_name=metric, value=value, recorded_at=when) - for metric, value in (("cpu_pct", cpu), ("mem_usage_bytes", mem))] + for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))] async def go(): async with app.db_sessionmaker() as s: @@ -78,7 +78,7 @@ def test_a_service_page_reads_every_task(app): placement_json=json.dumps([{"node_id": node, "running": 1}]), tasks_json=json.dumps(tasks), updated_at=now), m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running", - cpu_pct=50.0, mem_usage_bytes=100 << 20, service_name=name, + cpu_pct=50.0, mem_used_bytes=100 << 20, service_name=name, task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9), scraped_at=now), # Docker has removed the failed task's container: only its events are left. diff --git a/tests/integration/test_host_agent_docker_status.py b/tests/integration/test_host_agent_docker_status.py index 647d574..75ce5a2 100644 --- a/tests/integration/test_host_agent_docker_status.py +++ b/tests/integration/test_host_agent_docker_status.py @@ -30,7 +30,7 @@ def test_ingest_records_docker_status_and_clears_on_an_empty_answer(app): from sqlalchemy import text from steward.models.hosts import Host - web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0, + web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0, "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None} async def _go(): diff --git a/tests/integration/test_mcp_plugin_tools.py b/tests/integration/test_mcp_plugin_tools.py index 351fa17..7e69e16 100644 --- a/tests/integration/test_mcp_plugin_tools.py +++ b/tests/integration/test_mcp_plugin_tools.py @@ -70,10 +70,10 @@ def test_docker_and_agent_tools_over_seeded_rows(app): container_sql = text(""" INSERT INTO docker_containers - (host_id, name, container_id, image, status, cpu_pct, mem_pct, mem_usage_bytes, - mem_limit_bytes, restart_count, ports_json, started_at, scraped_at, health, - exit_code, oom_killed, compose_project, service_name, task_id) - VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, :rc, '[]', :started, :now, + (host_id, name, container_id, image, status, cpu_pct, mem_used_pct, mem_used_bytes, + mem_limit_bytes, mem_cache_bytes, restart_count, ports_json, started_at, scraped_at, + health, exit_code, oom_killed, compose_project, service_name, task_id) + VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, 5000, :rc, '[]', :started, :now, :health, :exit, false, :proj, :svc, :task) """) @@ -134,8 +134,8 @@ def test_docker_and_agent_tools_over_seeded_rows(app): "ts": now - timedelta(minutes=5 - i), "st": stream, "line": line}) # Container series live in the one metric store (#5752). for i, cpu in enumerate((10.0, 30.0, 50.0)): - for metric, value in (("cpu_pct", cpu), ("mem_pct", 20.0), - ("mem_usage_bytes", 1000.0)): + for metric, value in (("cpu_pct", cpu), ("mem_used_pct", 20.0), + ("mem_used_bytes", 1000.0)): await s.execute(text( "INSERT INTO plugin_metrics (id, source_module, resource_name, " "metric_name, value, recorded_at) " @@ -201,6 +201,10 @@ def test_docker_and_agent_tools_over_seeded_rows(app): c = await dtools.get_container(web, range="1h") assert c["host_name"] == host_name and c["status"] == "running" assert c["cpu_pct_history"]["max"] == 50.0 and c["cpu_pct_history"]["points"] == 3 + assert c["mem_used_pct_history"]["max"] == 20.0 + # File cache is in the one container's detail, not in the list rows (#6030). + assert c["mem_used_bytes"] == 1000 and c["mem_cache_bytes"] == 5000 + assert "mem_cache_bytes" not in proj[0] crashed = await dtools.get_container(job, host=host_name) assert [e["event"] for e in crashed["events"]] == ["die"] # A unique part of a long swarm task name is enough. diff --git a/tests/plugins/docker/test_docker_page.py b/tests/plugins/docker/test_docker_page.py index 9b5b1ab..a9c8dde 100644 --- a/tests/plugins/docker/test_docker_page.py +++ b/tests/plugins/docker/test_docker_page.py @@ -16,7 +16,7 @@ NOW = datetime.now(timezone.utc) def _c(name, **kw): base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None, - oom_killed=False, restart_count=0, cpu_pct=5.0, mem_pct=10.0, compose_project=None, + oom_killed=False, restart_count=0, cpu_pct=5.0, mem_used_pct=10.0, compose_project=None, service_name=None, ports_json=None, started_at=NOW - timedelta(hours=2), scraped_at=NOW - timedelta(seconds=20)) base.update(kw) diff --git a/tests/plugins/docker/test_mcp_tools.py b/tests/plugins/docker/test_mcp_tools.py index 7ca5e21..ad4cab9 100644 --- a/tests/plugins/docker/test_mcp_tools.py +++ b/tests/plugins/docker/test_mcp_tools.py @@ -15,8 +15,8 @@ NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc) def _c(**kw): base = dict(name="web", host_id="h1", image="nginx", status="running", health=None, - exit_code=None, oom_killed=False, restart_count=0, cpu_pct=1.0, mem_pct=2.0, - mem_usage_bytes=1, mem_limit_bytes=2, started_at=NOW - timedelta(hours=1), + exit_code=None, oom_killed=False, restart_count=0, cpu_pct=1.0, mem_used_pct=2.0, + mem_used_bytes=1, mem_limit_bytes=2, started_at=NOW - timedelta(hours=1), compose_project=None, service_name=None, scraped_at=NOW - timedelta(seconds=30)) base.update(kw) return NS(**base) diff --git a/tests/plugins/docker/test_overview.py b/tests/plugins/docker/test_overview.py index 09dd195..8ffbe64 100644 --- a/tests/plugins/docker/test_overview.py +++ b/tests/plugins/docker/test_overview.py @@ -265,7 +265,7 @@ def _ctr(**kw): base = dict(name="plex", host_id="h1", image="plexinc/pms:1.40", status="exited", health=None, exit_code=137, oom_killed=True, restart_count=2, compose_project="media", service_name=None, node_id=None, scraped_at=NOW - timedelta(seconds=30), - started_at=NOW - timedelta(hours=3)) + started_at=NOW - timedelta(hours=3), mem_cache_bytes=None) base.update(kw) return NS(**base) @@ -312,6 +312,18 @@ async def test_a_running_swarm_task_reads_under_its_service(app): assert "
Swarm node
" in page and "docker02" in page +async def test_file_cache_is_told_on_the_container_page_apart_from_its_memory(app): + """#6030: its memory figure is what its processes hold; the file cache + charged to it is shown here, said to be given back, and is "—" while + nothing reports it.""" + running = _ctr(status="running", exit_code=0, oom_killed=False, mem_cache_bytes=180 << 30) + page = await _container_page(app, running, exit_failed=False, uptime="3h") + assert "
File cache
180.0 GiB" in page + assert "kept in memory until the host needs it back; not counted in its memory" in page + stopped = await _container_page(app, _ctr()) + assert '
File cache
—
' in stopped + + async def test_a_service_log_line_names_its_task(app): lines = [{"ts": NOW, "stream": "stderr", "line": "boom", "task": f"1 · {TASK[:10]}"}] async with app.test_request_context("/plugins/docker/service/web/logs/lines"): @@ -335,7 +347,7 @@ def _swarm_svc(name, placement): def _task(name, service, node, cpu, mem, status="running"): return NS(name=name, service_name=service, node_id=node, status=status, cpu_pct=cpu, - mem_usage_bytes=mem) + mem_used_bytes=mem) def test_each_node_lists_what_runs_on_it_and_what_that_uses(): diff --git a/tests/plugins/docker/test_service.py b/tests/plugins/docker/test_service.py index 6efdcb9..641bbff 100644 --- a/tests/plugins/docker/test_service.py +++ b/tests/plugins/docker/test_service.py @@ -81,12 +81,12 @@ def test_task_label_names_the_service_and_slot(): def test_usage_reads_cpu_in_cores_and_memory_against_a_real_limit(): gib = 1024 ** 3 - c = NS(cpu_pct=150.0, mem_usage_bytes=gib, mem_limit_bytes=16 * gib, mem_pct=6.25) + c = NS(cpu_pct=150.0, mem_used_bytes=gib, mem_limit_bytes=16 * gib, mem_used_pct=6.25) u = svc.usage(c, 4, 16 * gib) assert u["cores"] == 1.5 and u["cpu_share"] == 37.5 # A limit of the host's whole memory is Docker saying "no limit". assert u["mem_limit"] is None and u["mem_of"] == "the host's memory" - limited = svc.usage(NS(cpu_pct=None, mem_usage_bytes=gib, mem_limit_bytes=2 * gib, mem_pct=50.0), 4, 16 * gib) + limited = svc.usage(NS(cpu_pct=None, mem_used_bytes=gib, mem_limit_bytes=2 * gib, mem_used_pct=50.0), 4, 16 * gib) assert limited["cores"] is None and limited["mem_limit"] == 2 * gib and limited["mem_of"] == "its limit" # Unknown host size: cores still read, no share to grade against. assert svc.usage(c, None, None)["cpu_share"] is None diff --git a/tests/plugins/docker/test_swarm_view.py b/tests/plugins/docker/test_swarm_view.py index 8cc2934..6dc8db3 100644 --- a/tests/plugins/docker/test_swarm_view.py +++ b/tests/plugins/docker/test_swarm_view.py @@ -17,7 +17,7 @@ NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc) def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0, task=None, started=None): return SimpleNamespace(host_id=host_id, name=name, service_name=service, - node_id=node, status=status, cpu_pct=cpu, mem_pct=2.0, + node_id=node, status=status, cpu_pct=cpu, mem_used_pct=2.0, health=None, restart_count=0, container_id=name, task_id=task, started_at=started) diff --git a/tests/plugins/host_agent/test_agent_docker.py b/tests/plugins/host_agent/test_agent_docker.py index 901c0b2..a600ae8 100644 --- a/tests/plugins/host_agent/test_agent_docker.py +++ b/tests/plugins/host_agent/test_agent_docker.py @@ -50,16 +50,41 @@ def test_docker_cpu_pct_guards_bad_stats(): }) == 0.0 -def test_docker_mem_subtracts_cache(): +def test_docker_mem_is_process_memory_not_file_cache(): + # cgroup v2, as curator_app read on 2026-10-17 (#6030): 315 GB charged to + # it, 2.2 GB its own; Docker's working set (usage − inactive_file) said 191 GB. stats = {"memory_stats": { - "usage": 200 * 1024 * 1024, - "limit": 1000 * 1024 * 1024, - "stats": {"cache": 50 * 1024 * 1024}, + "usage": 314_896_379_904, + "limit": 405_493_768_192, + "stats": {"anon": 2_241_888_256, "shmem": 106_496, "file": 312_003_436_544, + "active_file": 188_779_880_448, "inactive_file": 123_223_449_600}, }} - usage, limit, pct = a._docker_mem(stats) - assert usage == 150 * 1024 * 1024 # working set = usage − cache - assert limit == 1000 * 1024 * 1024 - assert pct == 15.0 + used, limit, pct, cache = a._docker_mem(stats) + assert used == 2_241_888_256 + 106_496 + assert limit == 405_493_768_192 + assert pct == 0.55 + assert cache == 312_003_436_544 - 106_496 # shmem is in file; counted as used, not cache + + +def test_docker_mem_counts_shared_memory_as_used(): + # Postgres keeps its shared buffers in shmem: not anon, but not reclaimable. + mib = 1024 * 1024 + stats = {"memory_stats": {"usage": 900 * mib, "limit": 1000 * mib, + "stats": {"anon": 100 * mib, "shmem": 400 * mib, "file": 700 * mib}}} + assert a._docker_mem(stats) == (500 * mib, 1000 * mib, 50.0, 300 * mib) + + +def test_docker_mem_reads_cgroup_v1(): + # v1's rss leaves shmem out and its cache has it in. + mib = 1024 * 1024 + stats = {"memory_stats": {"usage": 200 * mib, "limit": 1000 * mib, + "stats": {"total_rss": 120 * mib, "total_shmem": 30 * mib, + "total_cache": 80 * mib, "rss": 1, "cache": 1}}} + assert a._docker_mem(stats) == (150 * mib, 1000 * mib, 15.0, 50 * mib) + + +def test_docker_mem_without_stats_is_zero(): + assert a._docker_mem({}) == (0, 0, 0.0, 0) def test_docker_ports_keeps_only_published(): @@ -162,7 +187,8 @@ def test_collect_docker_assembles_container(monkeypatch): "precpu_stats": {"cpu_usage": {"total_usage": 100_000_000}, "system_cpu_usage": 1_000_000_000}, "memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024, - "stats": {"cache": 50 * 1024 * 1024}}, + "stats": {"anon": 140 * 1024 * 1024, "shmem": 10 * 1024 * 1024, + "file": 60 * 1024 * 1024}}, "networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}}, "blkio_stats": {"io_service_bytes_recursive": [ {"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]}, @@ -187,7 +213,8 @@ def test_collect_docker_assembles_container(monkeypatch): assert c["image"] == "nginx:latest" assert c["status"] == "running" assert c["cpu_pct"] == 40.0 - assert c["mem_pct"] == 15.0 + assert c["mem_used_bytes"] == 150 * 1024 * 1024 and c["mem_used_pct"] == 15.0 + assert c["mem_cache_bytes"] == 50 * 1024 * 1024 assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] # When it last started (inspect), not when it was created (2020). assert c["started_at"] == "2023-11-14T22:13:20.123456+00:00"