feat(docker): uptime from StartedAt, a restart is an event, host totals count physical devices only (#5153)
- Agent: started_at comes from inspect's State.StartedAt (nanoseconds
trimmed; Docker's zero time is "never started"), not the list's Created,
so a restarted container's uptime counts from its restart.
- A rising restart_count between two snapshots emits a "restart" event
("restart #3", or "3 restarts, now #5"), said instead of a start; a
restart then a death gives both. A count that falls (recreated under the
same name) says nothing. The timeline draws it as ↻.
- Network and disk I/O count what the kernel calls physical: interfaces
not under /sys/devices/virtual/net (bridges, docker0, veth, vxlan, tun),
block devices with a device behind them (not dm, md, zram), so a byte is
not counted twice. Where /sys calls everything virtual (an agent in LXC)
the name-filtered set is kept rather than reporting no traffic.
- Agent tests updated for #5152's (status, detail, containers) and
(is_manager, topology) shapes; the no-Docker disk test now states the
case it was passing for by accident.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+48
-11
@@ -91,6 +91,10 @@ OS_RELEASE_PATH = "/etc/os-release"
|
||||
NETDEV_PATH = "/proc/net/dev"
|
||||
DISKSTATS_PATH = "/proc/diskstats"
|
||||
HWMON_DIR = "/sys/class/hwmon"
|
||||
# Where the kernel lists virtual network devices (bridges, veth, docker0,
|
||||
# vxlan, tun, VLANs, bonds) and the block devices (a disk has a `device`).
|
||||
SYS_VIRTUAL_NET_DIR = "/sys/devices/virtual/net"
|
||||
SYS_BLOCK_DIR = "/sys/block"
|
||||
PSI_DIR = "/proc/pressure"
|
||||
|
||||
# Partition names to drop from disk I/O (we report whole-disk throughput only).
|
||||
@@ -259,10 +263,34 @@ def default_mounts() -> list[str]:
|
||||
return mounts or ["/"]
|
||||
|
||||
|
||||
def _physical_only(found: dict, is_virtual) -> dict:
|
||||
"""The entries the kernel says are physical, so a host total counts each
|
||||
byte once (a bridge's or an LVM volume's traffic is also counted on the
|
||||
NIC or disk beneath it). When /sys calls every one of them virtual (an
|
||||
agent inside LXC, where eth0 is a veth), they are all kept: a host
|
||||
reporting no traffic at all would be wrong, not filtered."""
|
||||
physical = {k: v for k, v in found.items() if not is_virtual(k)}
|
||||
return physical or found
|
||||
|
||||
|
||||
def _virtual_net(name: str) -> bool:
|
||||
"""Whether the kernel lists this interface as virtual; False if /sys can't say."""
|
||||
return os.path.isdir(SYS_VIRTUAL_NET_DIR) and os.path.exists(
|
||||
os.path.join(SYS_VIRTUAL_NET_DIR, name))
|
||||
|
||||
|
||||
def _virtual_disk(name: str) -> bool:
|
||||
"""Whether this block device has no disk behind it (dm, md, zram, nbd);
|
||||
False if /sys can't say."""
|
||||
return os.path.isdir(SYS_BLOCK_DIR) and not os.path.exists(
|
||||
os.path.join(SYS_BLOCK_DIR, name, "device"))
|
||||
|
||||
|
||||
def collect_net_raw() -> dict:
|
||||
"""Return {iface: (rx_bytes, tx_bytes)} cumulative counters from /proc/net/dev.
|
||||
|
||||
Skips loopback and veth* (container) interfaces to cut noise. Counters are
|
||||
Physical interfaces only (see _physical_only); loopback and veth* are
|
||||
skipped by name too, for a host whose /sys can't be read. Counters are
|
||||
monotonic; per-second rates are derived from deltas in build_sample().
|
||||
"""
|
||||
out: dict = {}
|
||||
@@ -284,14 +312,15 @@ def collect_net_raw() -> dict:
|
||||
out[name] = (int(f[0]), int(f[8])) # rx_bytes, tx_bytes
|
||||
except ValueError:
|
||||
continue
|
||||
return out
|
||||
return _physical_only(out, _virtual_net)
|
||||
|
||||
|
||||
def collect_diskio_raw() -> dict:
|
||||
"""Return {device: (read_bytes, write_bytes)} cumulative from /proc/diskstats.
|
||||
|
||||
Whole disks only — loop/ram/sr/fd and partitions are skipped. Sectors are
|
||||
512 bytes. Counters are monotonic; rates derived from deltas.
|
||||
Physical whole disks only (see _physical_only): loop/ram/sr/fd and
|
||||
partitions are skipped by name too. Sectors are 512 bytes. Counters are
|
||||
monotonic; rates derived from deltas.
|
||||
"""
|
||||
out: dict = {}
|
||||
try:
|
||||
@@ -309,7 +338,7 @@ def collect_diskio_raw() -> dict:
|
||||
out[name] = (int(f[5]) * 512, int(f[9]) * 512) # sectors_read, sectors_written
|
||||
except ValueError:
|
||||
continue
|
||||
return out
|
||||
return _physical_only(out, _virtual_disk)
|
||||
|
||||
|
||||
def collect_temps() -> list:
|
||||
@@ -563,6 +592,18 @@ def _docker_inspect_fields(insp: dict):
|
||||
return health, restart_count, exit_code, oom_killed
|
||||
|
||||
|
||||
def _docker_started_at(insp: dict):
|
||||
"""State.StartedAt as ISO-8601, or None for a container never started
|
||||
(Docker's zero time) or an inspect that failed. When the container last
|
||||
started, not when it was created: a restarted container's uptime counts
|
||||
from its restart."""
|
||||
raw = (((insp or {}).get("State") or {}).get("StartedAt") or "")
|
||||
if not raw or raw.startswith("0001-"):
|
||||
return None
|
||||
ts = _parse_log_ts(raw)
|
||||
return ts.isoformat() if ts else None
|
||||
|
||||
|
||||
def _docker_net_io(stats: dict):
|
||||
"""Return cumulative (net_rx, net_tx, blk_read, blk_write) bytes from stats.
|
||||
|
||||
@@ -596,7 +637,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
|
||||
|
||||
# Inspect every container (incl. stopped) — exit codes + restart counts only
|
||||
# live here, and stopped containers are exactly where exit_code matters.
|
||||
health = exit_code = None
|
||||
health = exit_code = insp = None
|
||||
restart_count = 0
|
||||
oom_killed = False
|
||||
try:
|
||||
@@ -616,11 +657,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
created = c.get("Created")
|
||||
started_at = (
|
||||
datetime.fromtimestamp(created, tz=timezone.utc).isoformat()
|
||||
if isinstance(created, (int, float)) else None
|
||||
)
|
||||
started_at = _docker_started_at(insp)
|
||||
record = {
|
||||
"name": name,
|
||||
"container_id": cid[:12],
|
||||
|
||||
Reference in New Issue
Block a user