bvandeusenandClaude Opus 5.5
bcd0563f39
- Agent: started_at comes from inspect's State.StartedAt (nanoseconds
trimmed; Docker's zero time is "never started"), not the list's Created,
so a restarted container's uptime counts from its restart.
- A rising restart_count between two snapshots emits a "restart" event
("restart #3", or "3 restarts, now #5"), said instead of a start; a
restart then a death gives both. A count that falls (recreated under the
same name) says nothing. The timeline draws it as ↻.
- Network and disk I/O count what the kernel calls physical: interfaces
not under /sys/devices/virtual/net (bridges, docker0, veth, vxlan, tun),
block devices with a device behind them (not dm, md, zram), so a byte is
not counted twice. Where /sys calls everything virtual (an agent in LXC)
the name-filtered set is kept rather than reporting no traffic.
- Agent tests updated for #5152's (status, detail, containers) and
(is_manager, topology) shapes; the no-Docker disk test now states the
case it was passing for by accident.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
328 lines
16 KiB
Python
328 lines
16 KiB
Python
# plugins/docker/models.py
|
|
from __future__ import annotations
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from sqlalchemy import (
|
|
BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
|
|
UniqueConstraint,
|
|
)
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
from steward.models.base import Base
|
|
|
|
|
|
class DockerContainer(Base):
|
|
"""Latest known state per container, scoped to the host that reported it.
|
|
|
|
Collection is per-host via the host agent, so container names are only
|
|
unique within a host — the natural key is (host_id, name). host_id is NOT
|
|
NULL: every container arrives through a host_agent ingest that resolves a
|
|
Host first. Deleting a host cascades its containers away.
|
|
"""
|
|
__tablename__ = "docker_containers"
|
|
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
|
)
|
|
name: Mapped[str] = mapped_column(String(255), primary_key=True)
|
|
container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
|
image: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
|
status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown")
|
|
# running | stopped | paused | exited | dead
|
|
cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
# BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes
|
|
# (~2.1 GB), which overflows int4 and fails the insert.
|
|
mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
|
# JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
|
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
scraped_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
|
|
# ── Enrichment (agent ≥ 1.4.0) ────────────────────────────────────────────
|
|
# Health/exit/restart come from `docker inspect` (not the list endpoint);
|
|
# exit_code is only meaningful for stopped containers.
|
|
health: Mapped[str | None] = mapped_column(String(16), nullable=True) # healthy|unhealthy|starting
|
|
exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
oom_killed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
# Grouping: compose project + swarm placement, read off container labels.
|
|
compose_project: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
service_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
task_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
node_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
# Cumulative-since-start I/O counters (BigInteger — they exceed 2^31 quickly).
|
|
net_rx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
net_tx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
blk_read_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
|
|
|
|
class DockerMetric(Base):
|
|
"""Time-series CPU/memory per container — one row per sample per running
|
|
container, scoped to the reporting host."""
|
|
__tablename__ = "docker_metrics"
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
|
)
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
|
|
)
|
|
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
scraped_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
index=True,
|
|
)
|
|
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
|
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
|
# BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
|
|
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
|
|
# Per-container history lookups filter on (host_id, container_name) then sort
|
|
# by time — a composite index keeps the rows() sparkline queries cheap.
|
|
__table_args__ = (
|
|
Index("ix_docker_metrics_host_container_time",
|
|
"host_id", "container_name", "scraped_at"),
|
|
)
|
|
|
|
|
|
class DockerMetricHourly(Base):
|
|
"""Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
|
|
|
|
Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
|
|
window; before deletion they're aggregated here so multi-day history stays
|
|
cheap to store and query. One row per (host, container, hour bucket); the
|
|
unique constraint lets retention upsert idempotently if it re-runs before the
|
|
raw rows are deleted. `bucket` is the hour-truncated sample time.
|
|
"""
|
|
__tablename__ = "docker_metrics_hourly"
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
|
)
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
|
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
|
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
|
|
__table_args__ = (
|
|
# One bucket per container per host — the conflict target for the
|
|
# idempotent rollup upsert; doubles as the history-query index.
|
|
UniqueConstraint("host_id", "container_name", "bucket",
|
|
name="uq_docker_metrics_hourly_bucket"),
|
|
Index("ix_docker_metrics_hourly_bucket", "bucket"),
|
|
)
|
|
|
|
|
|
class DockerEvent(Base):
|
|
"""Lifecycle events derived by diffing consecutive host snapshots.
|
|
|
|
The agent only reports current state, so Steward synthesises lifecycle by
|
|
comparing each new snapshot against the stored DockerContainer state for the
|
|
host: a container that appears → `start`; one that vanishes → `stop`; a
|
|
transition to a stopped status with a non-zero exit → `die`; an OOM-kill
|
|
flip → `oom`; a health status change → `health_change`. Scoped to the
|
|
reporting host (names are only unique within a host). `event` is a plain
|
|
string (no CHECK whitelist), matching how `status` is modelled on
|
|
DockerContainer — keeps the value set open without a migration per addition.
|
|
"""
|
|
__tablename__ = "docker_events"
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
|
)
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
event: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
# start | restart | stop | die | oom | health_change
|
|
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
# e.g. "exit 137", "healthy→unhealthy", image on start
|
|
at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
|
|
# Timeline lookups filter on (host_id, container_name) and sort by time;
|
|
# retention prunes by `at` alone — index both access paths.
|
|
__table_args__ = (
|
|
Index("ix_docker_events_host_container_time",
|
|
"host_id", "container_name", "at"),
|
|
Index("ix_docker_events_at", "at"),
|
|
)
|
|
|
|
|
|
class DockerLog(Base):
|
|
"""Container log lines pushed by the host agent — one row per line.
|
|
|
|
Time-series, scoped to the reporting host (container names are only unique
|
|
within a host, same identity as docker_metrics). The agent tails each running
|
|
container incrementally and folds new lines into its metrics push; `ts` is the
|
|
line's own Docker timestamp. Bounded by a per-container size+age ring
|
|
(retention), so a chatty container just keeps a shorter window rather than
|
|
growing without limit.
|
|
"""
|
|
__tablename__ = "docker_logs"
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
|
)
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
|
|
)
|
|
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
ts: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
stream: Mapped[str] = mapped_column(String(8), nullable=False, default="stdout")
|
|
# stdout | stderr
|
|
line: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
|
|
|
# Viewer filters on (host_id, container_name) and sorts by time; the ring
|
|
# prune walks the same key newest-first. A second index on `ts` alone serves
|
|
# the age-cutoff delete. Twin-index idiom, mirroring docker_events.
|
|
__table_args__ = (
|
|
Index("ix_docker_logs_host_container_time",
|
|
"host_id", "container_name", "ts"),
|
|
Index("ix_docker_logs_ts", "ts"),
|
|
)
|
|
|
|
|
|
class DockerSwarmService(Base):
|
|
"""A Swarm service as seen by a manager host: desired vs running replicas.
|
|
|
|
Manager-scoped — only a host that is a Swarm manager reports these, so a
|
|
single-manager cluster yields one row set keyed by that host. Service names
|
|
are unique within a swarm; we still key by (host_id, service_name) so two
|
|
managers reporting independently don't collide.
|
|
"""
|
|
__tablename__ = "docker_swarm_services"
|
|
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
|
)
|
|
service_name: Mapped[str] = mapped_column(String(255), primary_key=True)
|
|
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="replicated")
|
|
# replicated | global
|
|
desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
running: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
image: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
|
# task→node placement of running replicas: [{"node_id", "running"}], JSON.
|
|
# A manager sees every task, so this captures cross-node placement that the
|
|
# local docker_containers rows (this host only) can't.
|
|
placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
|
# The service's newest tasks, newest first, as the manager's agent shaped
|
|
# them: [{"task_id", "slot", "node_id", "desired_state", "state", "message",
|
|
# "error", "exit_code", "at"}], JSON. Swarm's own record of which tasks it
|
|
# replaced and why each one stopped; "[]" from agents older than 1.8.0.
|
|
tasks_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
|
|
|
|
class DockerDiskUsage(Base):
|
|
"""Per-host Docker disk usage summary from `/system/df` (current state).
|
|
|
|
One row per host (the agent re-reports the full summary each interval).
|
|
Reclaimable = bytes held by images no container references. Sizes are bytes
|
|
(BigInteger — image caches exceed 2^31 easily).
|
|
"""
|
|
__tablename__ = "docker_disk_usage"
|
|
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
|
)
|
|
layers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
images_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
images_reclaimable: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
containers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
volumes_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
build_cache_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
images_total: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
images_active: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
containers_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
volumes_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
|
|
|
|
class DockerImage(Base):
|
|
"""Per-host image storage record (the heavy hitters from `/system/df`).
|
|
|
|
Keyed (host_id, image_id); `containers` is the reference count (0 ⇒ the
|
|
image's size is reclaimable). Replaced wholesale per host on each report.
|
|
"""
|
|
__tablename__ = "docker_images"
|
|
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
|
)
|
|
image_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
repo_tag: Mapped[str] = mapped_column(String(512), nullable=False, default="<none>")
|
|
size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
shared_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
containers: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
|
|
|
|
class DockerSwarmNode(Base):
|
|
"""A Swarm node as seen by a manager host: role / availability / status."""
|
|
__tablename__ = "docker_swarm_nodes"
|
|
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
|
)
|
|
node_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
|
role: Mapped[str] = mapped_column(String(16), nullable=False, default="worker")
|
|
# manager | worker
|
|
availability: Mapped[str] = mapped_column(String(16), nullable=False, default="active")
|
|
# active | pause | drain
|
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown")
|
|
# ready | down | unknown
|
|
leader: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
|
|
|
|
class DockerHostStatus(Base):
|
|
"""Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
|
|
|
|
"ok" when it did; "unavailable" when there is no Docker to ask (no socket,
|
|
nothing listening); "error" when Docker is there and the call failed.
|
|
`detail` says why for the last two. While a host is not "ok" its container
|
|
rows are what Docker last said, not what is true now, and read as stale.
|
|
A host that never had Docker gets no row.
|
|
"""
|
|
__tablename__ = "docker_host_status"
|
|
|
|
host_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
|
)
|
|
status: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
# When the status last changed, and the agent's newest report.
|
|
since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|