feat(docker): uptime from StartedAt, a restart is an event, host totals count physical devices only (#5153)

- Agent: started_at comes from inspect's State.StartedAt (nanoseconds
  trimmed; Docker's zero time is "never started"), not the list's Created,
  so a restarted container's uptime counts from its restart.
- A rising restart_count between two snapshots emits a "restart" event
  ("restart #3", or "3 restarts, now #5"), said instead of a start; a
  restart then a death gives both. A count that falls (recreated under the
  same name) says nothing. The timeline draws it as ↻.
- Network and disk I/O count what the kernel calls physical: interfaces
  not under /sys/devices/virtual/net (bridges, docker0, veth, vxlan, tun),
  block devices with a device behind them (not dm, md, zram), so a byte is
  not counted twice. Where /sys calls everything virtual (an agent in LXC)
  the name-filtered set is kept rather than reporting no traffic.
- Agent tests updated for #5152's (status, detail, containers) and
  (is_manager, topology) shapes; the no-Docker disk test now states the
  case it was passing for by accident.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 18:45:57 -04:00
co-authored by Claude Opus 5.5
parent 8ceb983183
commit bcd0563f39
8 changed files with 170 additions and 32 deletions
+48 -11
View File
@@ -91,6 +91,10 @@ OS_RELEASE_PATH = "/etc/os-release"
NETDEV_PATH = "/proc/net/dev"
DISKSTATS_PATH = "/proc/diskstats"
HWMON_DIR = "/sys/class/hwmon"
# Where the kernel lists virtual network devices (bridges, veth, docker0,
# vxlan, tun, VLANs, bonds) and the block devices (a disk has a `device`).
SYS_VIRTUAL_NET_DIR = "/sys/devices/virtual/net"
SYS_BLOCK_DIR = "/sys/block"
PSI_DIR = "/proc/pressure"
# Partition names to drop from disk I/O (we report whole-disk throughput only).
@@ -259,10 +263,34 @@ def default_mounts() -> list[str]:
return mounts or ["/"]
def _physical_only(found: dict, is_virtual) -> dict:
"""The entries the kernel says are physical, so a host total counts each
byte once (a bridge's or an LVM volume's traffic is also counted on the
NIC or disk beneath it). When /sys calls every one of them virtual (an
agent inside LXC, where eth0 is a veth), they are all kept: a host
reporting no traffic at all would be wrong, not filtered."""
physical = {k: v for k, v in found.items() if not is_virtual(k)}
return physical or found
def _virtual_net(name: str) -> bool:
"""Whether the kernel lists this interface as virtual; False if /sys can't say."""
return os.path.isdir(SYS_VIRTUAL_NET_DIR) and os.path.exists(
os.path.join(SYS_VIRTUAL_NET_DIR, name))
def _virtual_disk(name: str) -> bool:
"""Whether this block device has no disk behind it (dm, md, zram, nbd);
False if /sys can't say."""
return os.path.isdir(SYS_BLOCK_DIR) and not os.path.exists(
os.path.join(SYS_BLOCK_DIR, name, "device"))
def collect_net_raw() -> dict:
"""Return {iface: (rx_bytes, tx_bytes)} cumulative counters from /proc/net/dev.
Skips loopback and veth* (container) interfaces to cut noise. Counters are
Physical interfaces only (see _physical_only); loopback and veth* are
skipped by name too, for a host whose /sys can't be read. Counters are
monotonic; per-second rates are derived from deltas in build_sample().
"""
out: dict = {}
@@ -284,14 +312,15 @@ def collect_net_raw() -> dict:
out[name] = (int(f[0]), int(f[8])) # rx_bytes, tx_bytes
except ValueError:
continue
return out
return _physical_only(out, _virtual_net)
def collect_diskio_raw() -> dict:
"""Return {device: (read_bytes, write_bytes)} cumulative from /proc/diskstats.
Whole disks only — loop/ram/sr/fd and partitions are skipped. Sectors are
512 bytes. Counters are monotonic; rates derived from deltas.
Physical whole disks only (see _physical_only): loop/ram/sr/fd and
partitions are skipped by name too. Sectors are 512 bytes. Counters are
monotonic; rates derived from deltas.
"""
out: dict = {}
try:
@@ -309,7 +338,7 @@ def collect_diskio_raw() -> dict:
out[name] = (int(f[5]) * 512, int(f[9]) * 512) # sectors_read, sectors_written
except ValueError:
continue
return out
return _physical_only(out, _virtual_disk)
def collect_temps() -> list:
@@ -563,6 +592,18 @@ def _docker_inspect_fields(insp: dict):
return health, restart_count, exit_code, oom_killed
def _docker_started_at(insp: dict):
"""State.StartedAt as ISO-8601, or None for a container never started
(Docker's zero time) or an inspect that failed. When the container last
started, not when it was created: a restarted container's uptime counts
from its restart."""
raw = (((insp or {}).get("State") or {}).get("StartedAt") or "")
if not raw or raw.startswith("0001-"):
return None
ts = _parse_log_ts(raw)
return ts.isoformat() if ts else None
def _docker_net_io(stats: dict):
"""Return cumulative (net_rx, net_tx, blk_read, blk_write) bytes from stats.
@@ -596,7 +637,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
# Inspect every container (incl. stopped) — exit codes + restart counts only
# live here, and stopped containers are exactly where exit_code matters.
health = exit_code = None
health = exit_code = insp = None
restart_count = 0
oom_killed = False
try:
@@ -616,11 +657,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
except (OSError, ValueError):
pass
created = c.get("Created")
started_at = (
datetime.fromtimestamp(created, tz=timezone.utc).isoformat()
if isinstance(created, (int, float)) else None
)
started_at = _docker_started_at(insp)
record = {
"name": name,
"container_id": cid[:12],